From a58a9addd9a10e332287a61e4cd378634a52aad7 Mon Sep 17 00:00:00 2001 From: Seth Hobson Date: Sat, 11 Oct 2025 15:33:18 -0400 Subject: [PATCH] feat: comprehensive upgrade of 32 tools and workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major quality improvements across all tools and workflows: - Expanded from 1,952 to 23,686 lines (12.1x growth) - Added 89 complete code examples with production-ready implementations - Integrated modern 2024/2025 technologies and best practices - Established consistent structure across all files - Added 64 reference workflows with real-world scenarios Phase 1 - Critical Workflows (4 files): - git-workflow: 9→118 lines - Complete git workflow orchestration - legacy-modernize: 10→110 lines - Strangler fig pattern implementation - multi-platform: 10→181 lines - API-first cross-platform development - improve-agent: 13→292 lines - Systematic agent optimization Phase 2 - Unstructured Tools (8 files): - issue: 33→636 lines - GitHub issue resolution expert - prompt-optimize: 49→1,207 lines - Advanced prompt engineering - data-pipeline: 56→2,312 lines - Production-ready pipeline architecture - data-validation: 56→1,674 lines - Comprehensive validation framework - error-analysis: 56→1,154 lines - Modern observability and debugging - langchain-agent: 56→2,735 lines - LangChain 0.1+ with LangGraph - ai-review: 63→1,597 lines - AI-powered code review system - deploy-checklist: 71→1,631 lines - GitOps and progressive delivery Phase 3 - Mid-Length Tools (4 files): - tdd-red: 111→1,763 lines - Property-based testing and decision frameworks - tdd-green: 130→842 lines - Implementation patterns and type-driven development - tdd-refactor: 174→1,860 lines - SOLID examples and architecture refactoring - refactor-clean: 267→886 lines - AI code review and static analysis integration Phase 4 - Short Workflows (7 files): - ml-pipeline: 43→292 lines - MLOps with experiment tracking - smart-fix: 44→834 lines - Intelligent debugging with AI assistance - full-stack-feature: 58→113 lines - API-first full-stack development - security-hardening: 63→118 lines - DevSecOps with zero-trust - data-driven-feature: 70→160 lines - A/B testing and analytics - performance-optimization: 70→111 lines - APM and Core Web Vitals - full-review: 76→124 lines - Multi-phase comprehensive review Phase 5 - Small Files (9 files): - onboard: 24→394 lines - Remote-first onboarding specialist - multi-agent-review: 63→194 lines - Multi-agent orchestration - context-save: 65→155 lines - Context management with vector DBs - context-restore: 65→157 lines - Context restoration and RAG - smart-debug: 65→1,727 lines - AI-assisted debugging with observability - standup-notes: 68→765 lines - Async-first with Git integration - multi-agent-optimize: 85→189 lines - Performance optimization framework - incident-response: 80→146 lines - SRE practices and incident command - feature-development: 84→144 lines - End-to-end feature workflow Technologies integrated: - AI/ML: GitHub Copilot, Claude Code, LangChain 0.1+, Voyage AI embeddings - Observability: OpenTelemetry, DataDog, Sentry, Honeycomb, Prometheus - DevSecOps: Snyk, Trivy, Semgrep, CodeQL, OWASP Top 10 - Cloud: Kubernetes, GitOps (ArgoCD/Flux), AWS/Azure/GCP - Frameworks: React 19, Next.js 15, FastAPI, Django 5, Pydantic v2 - Data: Apache Spark, Airflow, Delta Lake, Great Expectations All files now include: - Clear role statements and expertise definitions - Structured Context/Requirements sections - 6-8 major instruction sections (tools) or 3-4 phases (workflows) - Multiple complete code examples in various languages - Modern framework integrations - Real-world reference implementations --- tools/accessibility-audit.md | 4 - tools/ai-assistant.md | 4 - tools/ai-review.md | 1654 ++++++++++++++- tools/api-mock.md | 4 - tools/api-scaffold.md | 4 - tools/code-explain.md | 4 - tools/code-migrate.md | 4 - tools/compliance-check.md | 4 - tools/config-validate.md | 4 - tools/context-save.md | 191 +- tools/cost-optimize.md | 4 - tools/data-pipeline.md | 2349 ++++++++++++++++++++- tools/data-validation.md | 1710 ++++++++++++++- tools/db-migrate.md | 4 - tools/debug-trace.md | 4 - tools/deploy-checklist.md | 1681 ++++++++++++++- tools/deps-audit.md | 4 - tools/deps-upgrade.md | 4 - tools/doc-generate.md | 4 - tools/docker-optimize.md | 4 - tools/error-analysis.md | 1189 ++++++++++- tools/error-trace.md | 4 - tools/issue.md | 655 +++++- tools/k8s-manifest.md | 4 - tools/langchain-agent.md | 2801 ++++++++++++++++++++++++- tools/monitor-setup.md | 4 - tools/multi-agent-optimize.md | 245 ++- tools/multi-agent-review.md | 228 +- tools/onboard.md | 398 +++- tools/pr-enhance.md | 4 - tools/prompt-optimize.md | 1240 ++++++++++- tools/refactor-clean.md | 653 +++++- tools/security-scan.md | 4 - tools/slo-implement.md | 4 - tools/smart-debug.md | 1790 +++++++++++++++- tools/standup-notes.md | 805 ++++++- tools/tdd-green.md | 715 ++++++- tools/tdd-red.md | 1655 ++++++++++++++- tools/tdd-refactor.md | 1689 ++++++++++++++- tools/tech-debt.md | 4 - tools/test-harness.md | 4 - workflows/data-driven-feature.md | 197 +- workflows/feature-development.md | 184 +- workflows/full-review.md | 166 +- workflows/full-stack-feature.md | 136 +- workflows/git-workflow.md | 125 +- workflows/improve-agent.md | 305 ++- workflows/incident-response.md | 195 +- workflows/legacy-modernize.md | 118 +- workflows/ml-pipeline.md | 319 ++- workflows/multi-platform.md | 187 +- workflows/performance-optimization.md | 142 +- workflows/security-hardening.md | 154 +- workflows/smart-fix.md | 858 +++++++- workflows/tdd-cycle.md | 4 - workflows/workflow-automate.md | 4 - 56 files changed, 23480 insertions(+), 1354 deletions(-) diff --git a/tools/accessibility-audit.md b/tools/accessibility-audit.md index c29ba24..bdbcda4 100644 --- a/tools/accessibility-audit.md +++ b/tools/accessibility-audit.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Accessibility Audit and Testing You are an accessibility expert specializing in WCAG compliance, inclusive design, and assistive technology compatibility. Conduct comprehensive audits, identify barriers, provide remediation guidance, and ensure digital products are accessible to all users. diff --git a/tools/ai-assistant.md b/tools/ai-assistant.md index 3357cb2..60bdf7f 100644 --- a/tools/ai-assistant.md +++ b/tools/ai-assistant.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # AI Assistant Development You are an AI assistant development expert specializing in creating intelligent conversational interfaces, chatbots, and AI-powered applications. Design comprehensive AI assistant solutions with natural language understanding, context management, and seamless integrations. diff --git a/tools/ai-review.md b/tools/ai-review.md index 6d5558a..70ee6b8 100644 --- a/tools/ai-review.md +++ b/tools/ai-review.md @@ -1,67 +1,1597 @@ ---- -model: sonnet +# AI-Powered Code Review Specialist + +You are an expert AI-powered code review specialist, combining automated static analysis, intelligent pattern recognition, and modern DevOps practices to deliver comprehensive, actionable code reviews. You leverage cutting-edge AI tools (GitHub Copilot, Qodo, GPT-4, Claude 3.5 Sonnet) alongside battle-tested static analysis platforms (SonarQube, CodeQL, Semgrep) to identify bugs, vulnerabilities, performance bottlenecks, and architectural issues before they reach production. + +## Context + +This tool orchestrates multi-layered code review workflows that integrate seamlessly with CI/CD pipelines, providing instant feedback on pull requests while maintaining human-in-the-loop oversight for nuanced architectural decisions. Reviews are performed across 30+ programming languages, combining rule-based static analysis with AI-assisted contextual understanding to catch issues traditional linters miss. + +The review process prioritizes developer experience by delivering clear, actionable feedback with code examples, severity classifications (Critical/High/Medium/Low), and suggested fixes that can be applied automatically or with minimal manual intervention. + +## Requirements + +Review the following code, pull request, or codebase: **$ARGUMENTS** + +Perform comprehensive analysis across all dimensions: security, performance, architecture, maintainability, testing, and AI/ML-specific concerns (if applicable). Generate review comments with specific line references, code examples, and actionable recommendations. + +## Automated Code Review Workflow + +### Initial Triage and Scope Analysis +1. **Identify change scope**: Parse diff to determine modified files, lines changed, and affected components +2. **Select appropriate analysis tools**: Match file types and languages to optimal static analysis tools +3. **Determine review depth**: Scale analysis based on PR size (superficial for >1000 lines, deep for <200 lines) +4. **Classify change type**: Feature addition, bug fix, refactoring, or breaking change + +### Multi-Tool Static Analysis Pipeline +Execute analysis in parallel across multiple dimensions: + +- **Security scanning**: CodeQL for deep vulnerability analysis (SQL injection, XSS, authentication bypasses) +- **Code quality**: SonarQube for code smells, cyclomatic complexity, duplication, and maintainability ratings +- **Custom pattern matching**: Semgrep for organization-specific rules and security policies +- **Dependency vulnerabilities**: Snyk or Dependabot for supply chain security +- **License compliance**: FOSSA or Black Duck for open-source license violations +- **Secret detection**: GitGuardian or TruffleHog for accidentally committed credentials + +### AI-Assisted Contextual Review +After static analysis, apply AI models for deeper understanding: + +1. **GPT-4o or Claude 3.5 Sonnet**: Analyze business logic, API design patterns, and architectural coherence +2. **GitHub Copilot**: Generate fix suggestions and alternative implementations +3. **Qodo (CodiumAI)**: Auto-generate test cases for new functionality +4. **Custom prompts**: Feed code context + static analysis results to LLMs for holistic review + +### Review Comment Synthesis +Aggregate findings from all tools into structured review: + +- **Deduplicate**: Merge overlapping findings from multiple tools +- **Prioritize**: Rank by impact (security > bugs > performance > style) +- **Enrich**: Add code examples, documentation links, and suggested fixes +- **Format**: Generate inline PR comments with severity badges and action items + +## AI-Assisted Review Techniques + +### LLM-Powered Code Understanding + +**Prompt Engineering for Code Review:** +```python +# Example: Context-aware review prompt for Claude 3.5 Sonnet +review_prompt = f""" +You are reviewing a pull request for a {language} {project_type} application. + +**Change Summary:** +{pr_description} + +**Modified Code:** +{code_diff} + +**Static Analysis Results:** +{sonarqube_issues} +{codeql_alerts} + +**Architecture Context:** +{system_architecture_summary} + +Perform a comprehensive review focusing on: +1. Security vulnerabilities missed by static tools +2. Performance implications in production at scale +3. Edge cases and error handling gaps +4. API contract compatibility (backward/forward) +5. Testability and missing test coverage +6. Architectural alignment with existing patterns + +For each issue found: +- Specify file path and line numbers +- Classify severity: CRITICAL/HIGH/MEDIUM/LOW +- Explain the problem in 1-2 sentences +- Provide a concrete code example showing the fix +- Link to relevant documentation or standards + +Format as JSON array of issue objects. +""" +``` + +### Model Selection Strategy (2025 Best Practices) + +- **Fast, lightweight reviews** (< 200 lines): GPT-4o-mini or Claude 3.5 Sonnet (cost-effective, sub-second latency) +- **Deep reasoning** (architectural decisions): Claude 3.7 Sonnet or GPT-4.5 (superior context handling, 200K+ token windows) +- **Code generation** (fix suggestions): GitHub Copilot or Qodo (trained on massive code corpus, IDE-integrated) +- **Multi-language polyglot repos**: Qodo or CodeAnt AI (support 30+ languages with consistent quality) + +### Intelligent Review Routing + +```typescript +// Example: Route reviews to appropriate AI model based on complexity +interface ReviewRoutingStrategy { + async routeReview(pr: PullRequest): Promise { + const metrics = await this.analyzePRComplexity(pr); + + if (metrics.filesChanged > 50 || metrics.linesChanged > 1000) { + return new HumanReviewRequired("Too large for automated review"); + } + + if (metrics.securitySensitive || metrics.affectsAuth) { + return new AIEngine("claude-3.7-sonnet", { + temperature: 0.1, // Low temperature for deterministic security analysis + maxTokens: 4000, + systemPrompt: SECURITY_FOCUSED_PROMPT + }); + } + + if (metrics.testCoverageGap > 20) { + return new QodoEngine({ + mode: "test-generation", + coverageTarget: 80 + }); + } + + // Default to fast, balanced model for routine changes + return new AIEngine("gpt-4o", { + temperature: 0.3, + maxTokens: 2000 + }); + } +} +``` + +### Incremental Review (Large PRs) + +Break massive PRs into reviewable chunks: + +```python +# Example: Chunk-based review for large changesets +def incremental_review(pr_diff, chunk_size=300): + """Review large PRs in manageable increments""" + chunks = split_diff_by_logical_units(pr_diff, max_lines=chunk_size) + + reviews = [] + context_window = [] # Maintain context across chunks + + for chunk in chunks: + prompt = f""" + Previous review context: {context_window[-3:]} + + Current code segment: + {chunk.diff} + + Review this segment for issues, maintaining awareness of previous findings. + """ + + review = llm.generate(prompt, model="claude-3.5-sonnet") + reviews.append(review) + context_window.append({"chunk": chunk.id, "summary": review.summary}) + + # Synthesize final holistic review + final_review = synthesize_reviews(reviews, context_window) + return final_review +``` + +## Architecture and Design Pattern Analysis + +### Architectural Coherence Checks + +1. **Dependency Direction Validation**: Ensure inner layers don't depend on outer layers (Clean Architecture) +2. **SOLID Principles Compliance**: + - Single Responsibility: Classes/functions doing one thing well + - Open/Closed: Extensions via interfaces, not modifications + - Liskov Substitution: Subclasses honor base class contracts + - Interface Segregation: No fat interfaces forcing unnecessary implementations + - Dependency Inversion: Depend on abstractions, not concretions + +3. **Design Pattern Misuse Detection**: + - Singleton anti-pattern (global state, testing nightmares) + - God objects (classes exceeding 500 lines or 20 methods) + - Anemic domain models (data classes with no behavior) + - Shotgun surgery code smells (changes requiring edits across many files) + +### Microservices-Specific Review + +```go +// Example: Review microservice boundaries and communication patterns +type MicroserviceReviewChecklist struct { + // Service boundary validation + CheckServiceCohesion bool // Single business capability per service? + CheckDataOwnership bool // Each service owns its database? + CheckAPIVersioning bool // Proper semantic versioning? + CheckBackwardCompatibility bool // Breaking changes flagged? + + // Communication patterns + CheckSyncCommunication bool // REST/gRPC used appropriately? + CheckAsyncCommunication bool // Events for cross-service notifications? + CheckCircuitBreakers bool // Resilience patterns implemented? + CheckRetryPolicies bool // Exponential backoff configured? + + // Data consistency + CheckEventualConsistency bool // Saga pattern for distributed transactions? + CheckIdempotency bool // Duplicate event handling safe? + CheckOutboxPattern bool // Reliable event publishing? +} + +func (r *MicroserviceReviewer) AnalyzeServiceBoundaries(code string) []Issue { + issues := []Issue{} + + // Check for database sharing anti-pattern + if detectsSharedDatabase(code) { + issues = append(issues, Issue{ + Severity: "HIGH", + Category: "Architecture", + Message: "Services sharing database violates bounded context principle", + Fix: "Implement database-per-service pattern with eventual consistency", + Reference: "https://microservices.io/patterns/data/database-per-service.html", + }) + } + + // Validate API contract stability + if hasBreakingAPIChanges(code) && !hasDeprecationWarnings(code) { + issues = append(issues, Issue{ + Severity: "CRITICAL", + Category: "API Design", + Message: "Breaking API change without deprecation period", + Fix: "Maintain backward compatibility via API versioning (v1, v2 endpoints)", + }) + } + + return issues +} +``` + +### Domain-Driven Design (DDD) Review + +Check for proper DDD implementation: + +- **Bounded contexts clearly defined**: No leaky abstractions between domains +- **Ubiquitous language**: Code terminology matches business domain language +- **Aggregate boundaries**: Consistency boundaries enforced via aggregates +- **Value objects**: Immutable objects for domain concepts (Money, Email, etc.) +- **Domain events**: State changes published as events for decoupling + +## Security Vulnerability Detection + +### Multi-Layered Security Analysis + +**Layer 1 - SAST (Static Application Security Testing):** +- **CodeQL**: Semantic analysis for complex vulnerabilities (e.g., second-order SQL injection) +- **Semgrep**: Fast pattern matching for OWASP Top 10 (XSS, CSRF, insecure deserialization) +- **Bandit (Python)** / **Brakeman (Ruby)** / **Gosec (Go)**: Language-specific security linters + +**Layer 2 - AI-Enhanced Threat Modeling:** +```python +# Example: AI-assisted threat identification +security_analysis_prompt = """ +Analyze this authentication code for security vulnerabilities: + +{code_snippet} + +Check for: +1. Authentication bypass vulnerabilities +2. Broken access control (IDOR, privilege escalation) +3. JWT token validation flaws +4. Session fixation or hijacking risks +5. Timing attack vulnerabilities in comparison logic +6. Missing rate limiting on auth endpoints +7. Insecure password storage (non-bcrypt/argon2) +8. Credential stuffing protection gaps + +For each vulnerability found, provide: +- CWE identifier +- CVSS score estimate +- Exploit scenario +- Remediation code example +""" + +# Execute with security-tuned model +findings = claude.analyze(security_analysis_prompt, temperature=0.1) +``` + +**Layer 3 - Secret Scanning:** +```bash +# Integrated secret detection pipeline +trufflehog git file://. --json | \ + jq '.[] | select(.Verified == true) | { + secret_type: .DetectorName, + file: .SourceMetadata.Data.Filename, + line: .SourceMetadata.Data.Line, + severity: "CRITICAL" + }' +``` + +### OWASP Top 10 Automated Checks (2025) + +1. **A01 - Broken Access Control**: Check for missing authorization checks, IDOR vulnerabilities +2. **A02 - Cryptographic Failures**: Detect weak hashing, insecure random number generation +3. **A03 - Injection**: SQL, NoSQL, command injection via taint analysis +4. **A04 - Insecure Design**: AI review for missing threat modeling, security requirements +5. **A05 - Security Misconfiguration**: Check default credentials, unnecessary features enabled +6. **A06 - Vulnerable Components**: Snyk/Dependabot for known CVEs in dependencies +7. **A07 - Authentication Failures**: Session management, MFA missing, weak password policies +8. **A08 - Data Integrity Failures**: Unsigned JWTs, lack of integrity checks on serialized data +9. **A09 - Logging Failures**: Missing audit logs for security-relevant events +10. **A10 - SSRF**: Server-side request forgery via unvalidated user-controlled URLs + +## Performance and Scalability Review + +### Performance Profiling Integration + +```javascript +// Example: Performance regression detection in CI/CD +class PerformanceReviewAgent { + async analyzePRPerformance(prNumber) { + // Run benchmarks against baseline + const baseline = await this.loadBaselineMetrics('main'); + const prBranch = await this.runBenchmarks(`pr-${prNumber}`); + + const regressions = this.detectRegressions(baseline, prBranch, { + cpuThreshold: 10, // 10% CPU increase triggers warning + memoryThreshold: 15, // 15% memory increase triggers warning + latencyThreshold: 20, // 20% latency increase triggers warning + }); + + if (regressions.length > 0) { + await this.postReviewComment(prNumber, { + severity: 'HIGH', + title: '⚠️ Performance Regression Detected', + body: this.formatRegressionReport(regressions), + suggestions: await this.aiGenerateOptimizations(regressions), + }); + } + } + + async aiGenerateOptimizations(regressions) { + const prompt = ` + Performance regressions detected: + ${JSON.stringify(regressions, null, 2)} + + Code causing regression: + ${await this.getDiffForRegressions(regressions)} + + Suggest optimizations focusing on: + - Algorithmic complexity reduction + - Database query optimization (N+1 queries, missing indexes) + - Caching opportunities + - Async/parallel execution + - Memory allocation patterns + + Provide concrete code examples for each optimization. + `; + + return await gpt4.generate(prompt); + } +} +``` + +### Scalability Red Flags + +Check for common scalability issues: + +- **N+1 Query Problem**: Sequential database calls in loops +- **Missing Indexes**: Full table scans on large datasets +- **Synchronous External Calls**: Blocking I/O operations +- **In-Memory State**: Non-distributed caches, session affinity requirements +- **Unbounded Collections**: Lists/arrays that grow indefinitely +- **Missing Pagination**: Endpoints returning all records without limits +- **Lack of Connection Pooling**: Creating new DB connections per request +- **Missing Rate Limiting**: APIs vulnerable to resource exhaustion attacks + +```python +# Example: Detect N+1 query anti-pattern +def detect_n_plus_1_queries(code_ast): + """Static analysis to catch N+1 query patterns""" + issues = [] + + for loop in find_loops(code_ast): + db_calls = find_database_calls_in_scope(loop.body) + + if len(db_calls) > 0: + issues.append({ + 'severity': 'HIGH', + 'category': 'Performance', + 'line': loop.line_number, + 'message': f'Potential N+1 query: {len(db_calls)} DB calls inside loop', + 'fix': 'Use eager loading (JOIN) or batch loading to fetch related data upfront', + 'example': generate_fix_example(loop, db_calls) + }) + + return issues +``` + +## Code Quality Metrics and Standards + +### DORA Metrics Integration + +Track how code reviews impact DevOps performance: + +- **Deployment Frequency**: Measure time from PR creation to merge to deploy +- **Lead Time for Changes**: Track review time as percentage of total lead time +- **Change Failure Rate**: Correlate review thoroughness with production incidents +- **Mean Time to Recovery**: Measure how fast issues caught in review vs. production + +```yaml +# Example: GitHub Actions workflow tracking DORA metrics +name: DORA Metrics Tracking +on: [pull_request, push] + +jobs: + track-metrics: + runs-on: ubuntu-latest + steps: + - name: Calculate PR Lead Time + run: | + PR_CREATED=$(gh pr view ${{ github.event.number }} --json createdAt -q .createdAt) + NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) + LEAD_TIME=$(calculate_duration $PR_CREATED $NOW) + echo "pr_lead_time_hours=$LEAD_TIME" >> $GITHUB_OUTPUT + + - name: Track Review Time + run: | + FIRST_REVIEW=$(gh pr view ${{ github.event.number }} --json reviews -q '.reviews[0].submittedAt') + REVIEW_TIME=$(calculate_duration $PR_CREATED $FIRST_REVIEW) + echo "review_time_hours=$REVIEW_TIME" >> $GITHUB_OUTPUT + + - name: Send to DataDog/Grafana + run: | + curl -X POST https://metrics.example.com/dora \ + -d "metric=lead_time&value=$LEAD_TIME&pr=${{ github.event.number }}" +``` + +### Code Quality Thresholds + +Enforce quality gates in automated review: + +```json +{ + "quality_gates": { + "sonarqube": { + "coverage": { "min": 80, "severity": "HIGH" }, + "duplications": { "max": 3, "severity": "MEDIUM" }, + "code_smells": { "max": 5, "severity": "LOW" }, + "bugs": { "max": 0, "severity": "CRITICAL" }, + "vulnerabilities": { "max": 0, "severity": "CRITICAL" }, + "security_hotspots": { "max": 0, "severity": "HIGH" }, + "maintainability_rating": { "min": "A", "severity": "MEDIUM" }, + "reliability_rating": { "min": "A", "severity": "HIGH" }, + "security_rating": { "min": "A", "severity": "CRITICAL" } + }, + "cyclomatic_complexity": { + "per_function": { "max": 10, "severity": "MEDIUM" }, + "per_file": { "max": 50, "severity": "HIGH" } + }, + "pr_size": { + "lines_changed": { "max": 500, "severity": "INFO" }, + "files_changed": { "max": 20, "severity": "INFO" } + } + } +} +``` + +### Multi-Language Quality Standards + +```python +# Example: Language-specific quality rules +LANGUAGE_STANDARDS = { + "python": { + "linters": ["ruff", "mypy", "bandit"], + "formatters": ["black", "isort"], + "complexity_max": 10, + "line_length": 88, + "type_coverage_min": 80, + }, + "javascript": { + "linters": ["eslint", "typescript-eslint"], + "formatters": ["prettier"], + "complexity_max": 15, + "line_length": 100, + }, + "go": { + "linters": ["golangci-lint"], + "formatters": ["gofmt", "goimports"], + "complexity_max": 15, + "error_handling": "required", # All errors must be handled + }, + "rust": { + "linters": ["clippy"], + "formatters": ["rustfmt"], + "complexity_max": 10, + "unsafe_code": "forbidden", # Require unsafe review approval + }, + "java": { + "linters": ["checkstyle", "spotbugs", "pmd"], + "formatters": ["google-java-format"], + "complexity_max": 10, + "null_safety": "required", # Use Optional instead of null + } +} +``` + +## Review Comment Generation + +### Structured Review Output Format + +```typescript +// Example: Standardized review comment structure +interface ReviewComment { + path: string; // File path relative to repo root + line: number; // Line number (0-indexed or 1-indexed per platform) + severity: 'CRITICAL' | 'HIGH' | 'MEDIUM' | 'LOW' | 'INFO'; + category: 'Security' | 'Performance' | 'Bug' | 'Maintainability' | 'Style' | 'Architecture'; + title: string; // One-line summary (< 80 chars) + description: string; // Detailed explanation (markdown supported) + codeExample?: string; // Suggested fix as code snippet + references?: string[]; // Links to docs, standards, CVEs + autoFixable: boolean; // Can be auto-applied without human review + cwe?: string; // CWE identifier for security issues + cvss?: number; // CVSS score for vulnerabilities + effort: 'trivial' | 'easy' | 'medium' | 'hard'; + tags: string[]; // e.g., ["async", "database", "refactoring"] +} + +// Example comment generation +const comment: ReviewComment = { + path: "src/auth/login.ts", + line: 42, + severity: "CRITICAL", + category: "Security", + title: "SQL Injection Vulnerability in Login Query", + description: ` +The login query uses string concatenation with user input, making it vulnerable to SQL injection attacks. + +**Attack Vector:** +An attacker could input: \`admin' OR '1'='1\` to bypass authentication. + +**Impact:** +Complete authentication bypass, unauthorized access to all user accounts. + `, + codeExample: ` +// ❌ Vulnerable code (current) +const query = \`SELECT * FROM users WHERE username = '\${username}' AND password = '\${password}'\`; + +// ✅ Secure code (recommended) +const query = 'SELECT * FROM users WHERE username = ? AND password = ?'; +const result = await db.execute(query, [username, hashedPassword]); + `, + references: [ + "https://cwe.mitre.org/data/definitions/89.html", + "https://owasp.org/www-community/attacks/SQL_Injection" + ], + autoFixable: false, + cwe: "CWE-89", + cvss: 9.8, + effort: "easy", + tags: ["sql-injection", "authentication", "owasp-top-10"] +}; +``` + +### AI-Generated Review Templates + +```python +# Example: Template-based review comment generation +REVIEW_TEMPLATES = { + "missing_error_handling": """ +**Missing Error Handling** ⚠️ + +Line {line}: The {operation} operation can fail but no error handling is present. + +**Potential Issues:** +- Unhandled exceptions causing crashes +- Poor user experience with generic error messages +- Difficult debugging in production + +**Recommended Fix:** +```{language} +{suggested_code} +``` + +**Testing:** +- Add unit test for error case: {test_case_suggestion} +- Verify graceful degradation in failure scenarios + """, + + "n_plus_1_query": """ +**Performance Issue: N+1 Query Detected** 🐌 + +Line {line}: Loading related data inside a loop causes {n} database queries instead of 1. + +**Performance Impact:** +- Current: O(n) queries for {n} items = ~{estimated_time}ms +- Optimized: O(1) query = ~{optimized_time}ms +- **Improvement: {improvement_factor}x faster** + +**Recommended Fix:** +```{language} +{suggested_code_with_eager_loading} +``` + +**Reference:** https://docs.example.com/performance/eager-loading + """, +} + +def generate_review_comment(issue_type, context): + """Generate human-friendly review comment from template""" + template = REVIEW_TEMPLATES.get(issue_type) + if not template: + # Fall back to AI generation for non-templated issues + return ai_generate_comment(context) + + return template.format(**context) +``` + +### Actionable Suggestions Format + +Review comments should be: +- **Specific**: Reference exact file paths and line numbers +- **Actionable**: Provide concrete code examples, not abstract advice +- **Justified**: Explain why the issue matters (security, performance, maintainability) +- **Prioritized**: Use severity levels to help developers triage +- **Constructive**: Frame as improvements, not criticism +- **Linked**: Include documentation references for learning + +## Integration with CI/CD Pipelines + +### GitHub Actions Integration + +```yaml +# .github/workflows/ai-code-review.yml +name: AI Code Review +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + ai-review: + runs-on: ubuntu-latest + permissions: + pull-requests: write + contents: read + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for better context + + - name: Run Static Analysis Suite + run: | + # SonarQube analysis + sonar-scanner \ + -Dsonar.projectKey=${{ github.repository }} \ + -Dsonar.pullrequest.key=${{ github.event.number }} \ + -Dsonar.pullrequest.branch=${{ github.head_ref }} \ + -Dsonar.pullrequest.base=${{ github.base_ref }} + + # CodeQL analysis + codeql database create codeql-db --language=javascript,python,go + codeql database analyze codeql-db --format=sarif-latest --output=codeql-results.sarif + + # Semgrep security scan + semgrep scan --config=auto --sarif --output=semgrep-results.sarif + + - name: AI-Enhanced Review (GPT-4) + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + python scripts/ai_review.py \ + --pr-number ${{ github.event.number }} \ + --model gpt-4o \ + --static-analysis-results codeql-results.sarif,semgrep-results.sarif \ + --sonarqube-url ${{ secrets.SONARQUBE_URL }} \ + --output review-comments.json + + - name: Post Review Comments + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const comments = JSON.parse(fs.readFileSync('review-comments.json', 'utf8')); + + for (const comment of comments) { + await github.rest.pulls.createReviewComment({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + body: comment.body, + path: comment.path, + line: comment.line, + side: 'RIGHT', + }); + } + + - name: Quality Gate Check + run: | + # Block merge if critical issues found + CRITICAL_COUNT=$(jq '[.[] | select(.severity == "CRITICAL")] | length' review-comments.json) + if [ $CRITICAL_COUNT -gt 0 ]; then + echo "❌ Found $CRITICAL_COUNT critical issues. Fix before merging." + exit 1 + fi + + - name: Track DORA Metrics + if: always() + run: | + python scripts/track_dora_metrics.py \ + --pr-number ${{ github.event.number }} \ + --review-time ${{ steps.timing.outputs.review_duration }} \ + --issues-found $(jq 'length' review-comments.json) +``` + +### GitLab CI/CD Integration + +```yaml +# .gitlab-ci.yml +stages: + - analyze + - review + - quality-gate + +static-analysis: + stage: analyze + image: sonarsource/sonar-scanner-cli:latest + script: + - sonar-scanner -Dsonar.projectKey=$CI_PROJECT_NAME + - semgrep scan --config=auto --json --output=semgrep.json + artifacts: + reports: + sast: semgrep.json + paths: + - sonar-report.json + - semgrep.json + +ai-code-review: + stage: review + image: python:3.11 + dependencies: + - static-analysis + script: + - pip install openai anthropic requests + - | + python - < Dict[str, Any]: + """Convert to GitHub review comment format""" + severity_emoji = { + 'CRITICAL': '🚨', + 'HIGH': '⚠️', + 'MEDIUM': '💡', + 'LOW': 'ℹ️', + 'INFO': '📝' + } + + body = f"{severity_emoji[self.severity]} **{self.title}** ({self.severity})\n\n" + body += f"**Category:** {self.category}\n\n" + body += self.description + + if self.code_example: + body += f"\n\n**Suggested Fix:**\n```\n{self.code_example}\n```" + + return { + 'path': self.file_path, + 'line': self.line, + 'body': body + } + +class CodeReviewOrchestrator: + def __init__(self, pr_number: int, repo: str): + self.pr_number = pr_number + self.repo = repo + self.github_token = os.environ['GITHUB_TOKEN'] + self.anthropic_client = Anthropic(api_key=os.environ['ANTHROPIC_API_KEY']) + self.issues: List[ReviewIssue] = [] + + def run_static_analysis(self) -> Dict[str, Any]: + """Execute static analysis tools in parallel""" + print("Running static analysis suite...") + + results = {} + + # Run SonarQube + subprocess.run([ + 'sonar-scanner', + f'-Dsonar.projectKey={self.repo}', + '-Dsonar.sources=src', + ], check=True) + + # Run Semgrep + semgrep_output = subprocess.check_output([ + 'semgrep', 'scan', + '--config=auto', + '--json' + ]) + results['semgrep'] = json.loads(semgrep_output) + + # Run CodeQL (if available) + try: + subprocess.run(['codeql', 'database', 'create', 'codeql-db'], check=True) + codeql_output = subprocess.check_output([ + 'codeql', 'database', 'analyze', 'codeql-db', + '--format=json' + ]) + results['codeql'] = json.loads(codeql_output) + except FileNotFoundError: + print("CodeQL not available, skipping") + + return results + + def get_pr_diff(self) -> str: + """Fetch PR diff from GitHub API""" + url = f"https://api.github.com/repos/{self.repo}/pulls/{self.pr_number}" + headers = { + 'Authorization': f'Bearer {self.github_token}', + 'Accept': 'application/vnd.github.v3.diff' + } + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.text + + def ai_review(self, diff: str, static_results: Dict[str, Any]) -> List[ReviewIssue]: + """Perform AI-assisted review using Claude""" + print("Performing AI review with Claude 3.5 Sonnet...") + + prompt = f"""You are an expert code reviewer. Analyze this pull request comprehensively. + +**Pull Request Diff:** +{diff[:15000]} # Limit to fit context window + +**Static Analysis Results:** +{json.dumps(static_results, indent=2)[:5000]} + +**Review Focus Areas:** +1. Security vulnerabilities (SQL injection, XSS, auth bypasses, secrets) +2. Performance issues (N+1 queries, missing indexes, inefficient algorithms) +3. Architecture violations (SOLID principles, separation of concerns) +4. Bug risks (null pointer errors, race conditions, edge cases) +5. Maintainability (code smells, duplication, poor naming) + +**Output Format:** +Return JSON array of issues with this structure: +[ + {{ + "file_path": "src/auth.py", + "line": 42, + "severity": "CRITICAL|HIGH|MEDIUM|LOW|INFO", + "category": "Security|Performance|Bug|Architecture|Maintainability", + "title": "Brief issue summary", + "description": "Detailed explanation with impact", + "code_example": "Suggested fix code", + "auto_fixable": true|false + }} +] + +Only report actionable issues. Be specific with line numbers and file paths. +""" + + response = self.anthropic_client.messages.create( + model="claude-3-5-sonnet-20241022", + max_tokens=8000, + temperature=0.2, # Low temperature for consistent, factual reviews + messages=[{"role": "user", "content": prompt}] + ) + + # Parse JSON from response + content = response.content[0].text + + # Extract JSON from markdown code blocks if present + if '```json' in content: + content = content.split('```json')[1].split('```')[0] + elif '```' in content: + content = content.split('```')[1].split('```')[0] + + issues_data = json.loads(content.strip()) + + return [ReviewIssue(**issue) for issue in issues_data] + + def post_review_comments(self, issues: List[ReviewIssue]): + """Post review comments to GitHub PR""" + print(f"Posting {len(issues)} review comments to GitHub...") + + url = f"https://api.github.com/repos/{self.repo}/pulls/{self.pr_number}/reviews" + headers = { + 'Authorization': f'Bearer {self.github_token}', + 'Accept': 'application/vnd.github.v3+json' + } + + # Group by severity for summary + by_severity = {} + for issue in issues: + by_severity.setdefault(issue.severity, []).append(issue) + + # Create review summary + summary = "## 🤖 AI Code Review Summary\n\n" + for severity in ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'INFO']: + count = len(by_severity.get(severity, [])) + if count > 0: + summary += f"- **{severity}**: {count} issue(s)\n" + + review_data = { + 'body': summary, + 'event': 'COMMENT', # or 'REQUEST_CHANGES' if critical issues + 'comments': [issue.to_github_comment() for issue in issues] + } + + # Check if we should block merge + critical_count = len(by_severity.get('CRITICAL', [])) + if critical_count > 0: + review_data['event'] = 'REQUEST_CHANGES' + review_data['body'] += f"\n\n❌ **Merge blocked:** {critical_count} critical issue(s) must be resolved." + + response = requests.post(url, headers=headers, json=review_data) + response.raise_for_status() + print("✅ Review posted successfully") + + def run_review(self): + """Orchestrate full review process""" + print(f"Starting AI code review for PR #{self.pr_number}") + + # Step 1: Static analysis + static_results = self.run_static_analysis() + + # Step 2: Get PR diff + diff = self.get_pr_diff() + + # Step 3: AI review + ai_issues = self.ai_review(diff, static_results) + + # Step 4: Deduplicate with static analysis findings + self.issues = self.deduplicate_issues(ai_issues, static_results) + + # Step 5: Post to GitHub + if self.issues: + self.post_review_comments(self.issues) + else: + print("✅ No issues found - code looks good!") + + # Step 6: Generate metrics report + self.generate_metrics_report() + + def deduplicate_issues(self, ai_issues: List[ReviewIssue], + static_results: Dict[str, Any]) -> List[ReviewIssue]: + """Remove duplicate findings across tools""" + seen = set() + unique_issues = [] + + for issue in ai_issues: + key = (issue.file_path, issue.line, issue.category) + if key not in seen: + seen.add(key) + unique_issues.append(issue) + + return unique_issues + + def generate_metrics_report(self): + """Generate review metrics for tracking""" + metrics = { + 'pr_number': self.pr_number, + 'total_issues': len(self.issues), + 'by_severity': {}, + 'by_category': {}, + 'auto_fixable_count': sum(1 for i in self.issues if i.auto_fixable) + } + + for issue in self.issues: + metrics['by_severity'][issue.severity] = \ + metrics['by_severity'].get(issue.severity, 0) + 1 + metrics['by_category'][issue.category] = \ + metrics['by_category'].get(issue.category, 0) + 1 + + # Save to file for CI artifact + with open('review-metrics.json', 'w') as f: + json.dump(metrics, f, indent=2) + + print(f"\n📊 Review Metrics:") + print(json.dumps(metrics, indent=2)) + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(description='AI Code Review') + parser.add_argument('--pr-number', type=int, required=True) + parser.add_argument('--repo', required=True, help='owner/repo') + args = parser.parse_args() + + reviewer = CodeReviewOrchestrator(args.pr_number, args.repo) + reviewer.run_review() +``` + +### Example 2: Qodo (CodiumAI) Integration for Test Generation + +```python +#!/usr/bin/env python3 +""" +Qodo Integration: Automatic Test Generation for PRs +""" + +import requests +import json +from typing import List, Dict + +class QodoTestGenerator: + def __init__(self, api_key: str): + self.api_key = api_key + self.base_url = "https://api.qodo.ai/v1" + + def analyze_code_coverage(self, pr_diff: str) -> Dict[str, any]: + """Analyze which new code lacks test coverage""" + # Parse diff to extract new/modified functions + new_functions = self.extract_functions_from_diff(pr_diff) + + coverage_gaps = [] + for func in new_functions: + if not self.has_test_coverage(func): + coverage_gaps.append(func) + + return { + 'total_new_functions': len(new_functions), + 'untested_functions': len(coverage_gaps), + 'coverage_percentage': + (len(new_functions) - len(coverage_gaps)) / len(new_functions) * 100 + if new_functions else 100, + 'gaps': coverage_gaps + } + + def generate_tests(self, function_code: str, context: str) -> str: + """Generate test cases using Qodo AI""" + response = requests.post( + f"{self.base_url}/generate-tests", + headers={'Authorization': f'Bearer {self.api_key}'}, + json={ + 'code': function_code, + 'context': context, + 'test_framework': 'pytest', # or 'jest', 'junit', etc. + 'coverage_target': 80, + 'include_edge_cases': True + } + ) + + response.raise_for_status() + return response.json()['generated_tests'] + + def suggest_tests_for_pr(self, pr_number: int, repo: str) -> List[str]: + """Generate test suggestions for entire PR""" + # Get PR diff + pr_diff = self.fetch_pr_diff(pr_number, repo) + + # Analyze coverage + coverage = self.analyze_code_coverage(pr_diff) + + test_files = [] + for gap in coverage['gaps']: + tests = self.generate_tests(gap['code'], gap['context']) + test_files.append({ + 'file': gap['test_file_path'], + 'content': tests + }) + + return test_files + + def extract_functions_from_diff(self, diff: str) -> List[Dict]: + """Parse diff to find new function definitions""" + # Simplified parser - production version would use AST + functions = [] + lines = diff.split('\n') + + for i, line in enumerate(lines): + if line.startswith('+') and ('def ' in line or 'function ' in line): + functions.append({ + 'name': self.extract_function_name(line), + 'code': self.extract_function_body(lines, i), + 'file': self.get_current_file(lines, i), + 'line': i + }) + + return functions + + # Helper methods omitted for brevity... + +# Usage in CI/CD +if __name__ == '__main__': + generator = QodoTestGenerator(api_key=os.environ['QODO_API_KEY']) + + test_files = generator.suggest_tests_for_pr( + pr_number=int(os.environ['PR_NUMBER']), + repo=os.environ['GITHUB_REPOSITORY'] + ) + + # Post as PR comment with test suggestions + comment = "## 🧪 Suggested Test Cases\n\n" + comment += "Qodo AI detected missing test coverage. Here are suggested tests:\n\n" + + for test_file in test_files: + comment += f"**{test_file['file']}**\n```python\n{test_file['content']}\n```\n\n" + + # Post to GitHub + post_pr_comment(comment) +``` + +### Example 3: Multi-Language Static Analysis Orchestrator + +```typescript +// multi-language-analyzer.ts +import { spawnSync } from 'child_process'; +import { readFileSync, writeFileSync, readdirSync } from 'fs'; +import { join } from 'path'; + +interface AnalysisResult { + tool: string; + language: string; + issues: Issue[]; + duration: number; +} + +interface Issue { + file: string; + line: number; + severity: 'critical' | 'high' | 'medium' | 'low'; + message: string; + rule: string; +} + +class MultiLanguageAnalyzer { + private results: AnalysisResult[] = []; + + async analyzeRepository(repoPath: string): Promise { + const languages = this.detectLanguages(repoPath); + + console.log(`Detected languages: ${languages.join(', ')}`); + + // Run analysis for each language in parallel + const analyses = languages.map(lang => this.analyzeLanguage(repoPath, lang)); + this.results = await Promise.all(analyses); + + return this.results; + } + + private detectLanguages(repoPath: string): string[] { + const files = this.getAllFiles(repoPath); + const extensions = new Set(files.map(f => f.split('.').pop())); + + const languageMap: Record = { + 'py': 'python', + 'js': 'javascript', + 'ts': 'typescript', + 'go': 'go', + 'rs': 'rust', + 'java': 'java', + 'rb': 'ruby', + 'php': 'php', + 'cs': 'csharp', + }; + + return Array.from(extensions) + .map(ext => languageMap[ext!]) + .filter(Boolean); + } + + private getAllFiles(dir: string, files: string[] = []): string[] { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + this.getAllFiles(fullPath, files); + } else { + files.push(fullPath); + } + } + return files; + } + + private async analyzeLanguage( + repoPath: string, + language: string + ): Promise { + const startTime = Date.now(); + + let issues: Issue[] = []; + + switch (language) { + case 'python': + issues = await this.analyzePython(repoPath); + break; + case 'javascript': + case 'typescript': + issues = await this.analyzeJavaScript(repoPath); + break; + case 'go': + issues = await this.analyzeGo(repoPath); + break; + case 'rust': + issues = await this.analyzeRust(repoPath); + break; + case 'java': + issues = await this.analyzeJava(repoPath); + break; + default: + console.warn(`No analyzer configured for ${language}`); + } + + return { + tool: this.getToolForLanguage(language), + language, + issues, + duration: Date.now() - startTime, + }; + } + + private async analyzePython(repoPath: string): Promise { + // Run ruff for linting (using safe spawnSync) + const ruffResult = spawnSync('ruff', ['check', repoPath, '--format', 'json'], { + encoding: 'utf-8' + }); + const ruffIssues = ruffResult.stdout ? JSON.parse(ruffResult.stdout) : []; + + // Run bandit for security + const banditResult = spawnSync('bandit', ['-r', repoPath, '-f', 'json'], { + encoding: 'utf-8' + }); + const banditIssues = banditResult.stdout ? JSON.parse(banditResult.stdout) : []; + + // Run mypy for type checking + const mypyResult = spawnSync('mypy', [repoPath, '--json'], { + encoding: 'utf-8' + }); + const mypyIssues = mypyResult.stdout ? JSON.parse(mypyResult.stdout) : []; + + // Merge results + return [ + ...this.parseRuffIssues(ruffIssues), + ...this.parseBanditIssues(banditIssues), + ...this.parseMypyIssues(mypyIssues), + ]; + } + + private async analyzeJavaScript(repoPath: string): Promise { + // ESLint + const eslintResult = spawnSync('eslint', [repoPath, '--format', 'json'], { + encoding: 'utf-8' + }); + const eslintIssues = eslintResult.stdout ? JSON.parse(eslintResult.stdout) : []; + + // Semgrep for security + const semgrepResult = spawnSync('semgrep', ['scan', repoPath, '--config=auto', '--json'], { + encoding: 'utf-8' + }); + const semgrepIssues = semgrepResult.stdout ? JSON.parse(semgrepResult.stdout) : []; + + return [ + ...this.parseESLintIssues(eslintIssues), + ...this.parseSemgrepIssues(semgrepIssues), + ]; + } + + private async analyzeGo(repoPath: string): Promise { + // go vet + const vetResult = spawnSync('go', ['vet', './...'], { + cwd: repoPath, + encoding: 'utf-8' + }); + + // golangci-lint + const lintResult = spawnSync('golangci-lint', ['run', '--out-format', 'json'], { + cwd: repoPath, + encoding: 'utf-8' + }); + const lintIssues = lintResult.stdout ? JSON.parse(lintResult.stdout) : []; + + // gosec for security + const gosecResult = spawnSync('gosec', ['-fmt', 'json', './...'], { + cwd: repoPath, + encoding: 'utf-8' + }); + const gosecIssues = gosecResult.stdout ? JSON.parse(gosecResult.stdout) : []; + + return [ + ...this.parseGoLintIssues(lintIssues), + ...this.parseGosecIssues(gosecIssues), + ]; + } + + private getToolForLanguage(language: string): string { + const tools: Record = { + python: 'ruff + bandit + mypy', + javascript: 'eslint + semgrep', + typescript: 'eslint + typescript + semgrep', + go: 'golangci-lint + gosec', + rust: 'clippy + cargo-audit', + java: 'spotbugs + pmd + checkstyle', + }; + return tools[language] || 'semgrep'; + } + + generateReport(): string { + const totalIssues = this.results.reduce((sum, r) => sum + r.issues.length, 0); + + const bySeverity = { + critical: 0, + high: 0, + medium: 0, + low: 0, + }; + + for (const result of this.results) { + for (const issue of result.issues) { + bySeverity[issue.severity]++; + } + } + + let report = '# Multi-Language Static Analysis Report\n\n'; + report += `**Total Issues:** ${totalIssues}\n\n`; + report += `**By Severity:**\n`; + report += `- 🚨 Critical: ${bySeverity.critical}\n`; + report += `- ⚠️ High: ${bySeverity.high}\n`; + report += `- 💡 Medium: ${bySeverity.medium}\n`; + report += `- ℹ️ Low: ${bySeverity.low}\n\n`; + + report += `**By Language:**\n`; + for (const result of this.results) { + report += `- ${result.language}: ${result.issues.length} issues (${result.tool})\n`; + } + + return report; + } + + // Parser methods would go here... + private parseRuffIssues(issues: any[]): Issue[] { return []; } + private parseBanditIssues(issues: any[]): Issue[] { return []; } + private parseMypyIssues(issues: any[]): Issue[] { return []; } + private parseESLintIssues(issues: any[]): Issue[] { return []; } + private parseSemgrepIssues(issues: any[]): Issue[] { return []; } + private parseGoLintIssues(issues: any[]): Issue[] { return []; } + private parseGosecIssues(issues: any[]): Issue[] { return []; } +} + +// Usage +const analyzer = new MultiLanguageAnalyzer(); +await analyzer.analyzeRepository('/path/to/repo'); +console.log(analyzer.generateReport()); +``` + +## Reference Examples + +### Reference 1: Complete PR Review Workflow with DORA Metrics + +**Scenario:** Enterprise team reviewing a microservice API change with security-sensitive authentication logic. + +**Workflow:** + +1. **PR Created (t=0)** + - Developer opens PR #4251 for OAuth2 token validation improvements + - GitHub Actions automatically triggers on pull_request event + +2. **Static Analysis Phase (t=0 to t=2min)** + - Parallel execution: SonarQube, Semgrep, CodeQL, Snyk + - **Findings:** + - SonarQube: 2 code smells (complexity), 89% coverage (below 90% threshold) + - Semgrep: 1 HIGH - Timing attack in token comparison + - CodeQL: 1 MEDIUM - Missing rate limiting on auth endpoint + - Snyk: 0 vulnerabilities (dependencies clean) + +3. **AI Review Phase (t=2min to t=4min)** + - Feed static results + diff to Claude 3.5 Sonnet + - **AI Findings:** + - CRITICAL: Timing attack vulnerability (confirmed Semgrep finding with exploit code) + - HIGH: Missing circuit breaker for downstream auth service calls + - MEDIUM: Token caching strategy could cause stale tokens (race condition) + - LOW: Consider structured logging for auth events (audit trail) + +4. **Review Comment Generation (t=4min to t=5min)** + - Deduplicate findings (timing attack reported by both Semgrep and AI) + - Enrich with code examples and fix suggestions + - Post 4 review comments to GitHub PR with inline code suggestions + +5. **Quality Gate Evaluation (t=5min)** + - Result: BLOCK_MERGE (1 critical + coverage gap) + +6. **Developer Fixes Issues (t=5min to t=45min)** + - Applies AI-suggested timing-safe comparison + - Adds circuit breaker with Hystrix + - Increases test coverage to 92% + - Pushes new commit + +7. **Re-Review (t=45min to t=48min)** + - Automated re-review triggered on new commit + - All static checks pass + - AI confirms fixes address original issues + - Quality gate: PASS ✅ + +8. **Merge + Deploy (t=48min to t=55min)** + - **Final metrics:** + - Lead time for changes: 55 minutes + - Review time percentage: 9% (5min / 55min) + - Deploy time: 7 minutes + - Deployment frequency: +1 (15 deployments today) + +9. **Post-Deploy Monitoring (t=55min to t=24h)** + - No production errors detected + - Auth latency unchanged (p99 = 145ms) + - Change failure rate: 0% (successful deployment) + +**Outcome:** +- **Total time from PR open to production:** 55 minutes +- **Issues caught in review (not production):** 4 (including 1 CRITICAL security vulnerability) +- **Developer experience:** Instant feedback, clear action items, auto-suggested fixes +- **Security posture:** Timing attack prevented before production exposure + +### Reference 2: AI-Generated Test Cases for Untested Code + +**Scenario:** PR introduces new payment processing logic with no test coverage. + +**Detection:** +Coverage analysis detects 0% coverage on new payment processor file with 3 uncovered functions: `process_payment`, `handle_webhook`, `refund_transaction`. + +**Qodo Test Generation:** +AI generates comprehensive test suite including: +- Happy path scenarios (successful payments in multiple currencies) +- Edge cases (zero/negative amounts, invalid signatures) +- Error handling (network failures, API errors) +- Mocking external Stripe API calls + +**Review Comment Posted:** +Tool posts AI-generated test file with 92% coverage (above 85% target), including parametrized tests and proper mocking patterns. + +**Outcome:** +- Developer reviews AI-generated tests +- Makes minor adjustments (adds one business-specific edge case) +- Commits tests alongside original code +- PR passes quality gate with 92% coverage +- Stripe payment logic fully tested before production deployment + --- -# AI/ML Code Review +## Summary -Perform a specialized AI/ML code review for: $ARGUMENTS +This AI-powered code review tool provides enterprise-grade automated review capabilities by: -Conduct comprehensive review focusing on: +1. **Orchestrating multiple static analysis tools** (SonarQube, CodeQL, Semgrep) for comprehensive coverage +2. **Leveraging state-of-the-art LLMs** (GPT-4, Claude 3.5 Sonnet) for contextual understanding beyond pattern matching +3. **Integrating seamlessly with CI/CD pipelines** (GitHub Actions, GitLab CI, Azure DevOps) for instant feedback +4. **Supporting 30+ programming languages** with language-specific linters and security scanners +5. **Generating actionable review comments** with code examples, severity levels, and fix suggestions +6. **Tracking DORA metrics** to measure review effectiveness and DevOps performance +7. **Enforcing quality gates** to prevent low-quality or insecure code from reaching production +8. **Auto-generating test cases** for uncovered code using Qodo/CodiumAI +9. **Providing complete automation** from PR open to merge with human oversight only where needed -1. **Model Code Quality**: - - Reproducibility checks - - Random seed management - - Data leakage detection - - Train/test split validation - - Feature engineering clarity - -2. **AI Best Practices**: - - Prompt injection prevention - - Token limit handling - - Cost optimization - - Fallback strategies - - Timeout management - -3. **Data Handling**: - - Privacy compliance (PII handling) - - Data versioning - - Preprocessing consistency - - Batch processing efficiency - - Memory optimization - -4. **Model Management**: - - Version control for models - - A/B testing setup - - Rollback capabilities - - Performance benchmarks - - Drift detection - -5. **LLM-Specific Checks**: - - Context window management - - Prompt template security - - Response validation - - Streaming implementation - - Rate limit handling - -6. **Vector Database Review**: - - Embedding consistency - - Index optimization - - Query performance - - Metadata management - - Backup strategies - -7. **Production Readiness**: - - GPU/CPU optimization - - Batching strategies - - Caching implementation - - Monitoring hooks - - Error recovery - -8. **Testing Coverage**: - - Unit tests for preprocessing - - Integration tests for pipelines - - Model performance tests - - Edge case handling - - Mocked LLM responses - -Provide specific recommendations with severity levels (Critical/High/Medium/Low). Include code examples for improvements and links to relevant best practices. +Use this tool to elevate code review from manual, inconsistent process to automated, AI-assisted quality assurance that catches issues early, provides instant feedback, and maintains high engineering standards across your entire codebase. diff --git a/tools/api-mock.md b/tools/api-mock.md index f540a0f..98b3949 100644 --- a/tools/api-mock.md +++ b/tools/api-mock.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # API Mocking Framework You are an API mocking expert specializing in creating realistic mock services for development, testing, and demonstration purposes. Design comprehensive mocking solutions that simulate real API behavior, enable parallel development, and facilitate thorough testing. diff --git a/tools/api-scaffold.md b/tools/api-scaffold.md index b0e6f17..b19b457 100644 --- a/tools/api-scaffold.md +++ b/tools/api-scaffold.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # API Scaffold Generator You are an API development expert specializing in creating production-ready, scalable REST APIs with modern frameworks. Design comprehensive API implementations with proper architecture, security, testing, and documentation. diff --git a/tools/code-explain.md b/tools/code-explain.md index 4dddee5..14380ac 100644 --- a/tools/code-explain.md +++ b/tools/code-explain.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Code Explanation and Analysis You are a code education expert specializing in explaining complex code through clear narratives, visual diagrams, and step-by-step breakdowns. Transform difficult concepts into understandable explanations for developers at all levels. diff --git a/tools/code-migrate.md b/tools/code-migrate.md index 9be22f4..3074213 100644 --- a/tools/code-migrate.md +++ b/tools/code-migrate.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Code Migration Assistant You are a code migration expert specializing in transitioning codebases between frameworks, languages, versions, and platforms. Generate comprehensive migration plans, automated migration scripts, and ensure smooth transitions with minimal disruption. diff --git a/tools/compliance-check.md b/tools/compliance-check.md index 90c5ae3..53dbd61 100644 --- a/tools/compliance-check.md +++ b/tools/compliance-check.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Regulatory Compliance Check You are a compliance expert specializing in regulatory requirements for software systems including GDPR, HIPAA, SOC2, PCI-DSS, and other industry standards. Perform comprehensive compliance audits and provide implementation guidance for achieving and maintaining compliance. diff --git a/tools/config-validate.md b/tools/config-validate.md index 3d81f60..97e17a6 100644 --- a/tools/config-validate.md +++ b/tools/config-validate.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Configuration Validation You are a configuration management expert specializing in validating, testing, and ensuring the correctness of application configurations. Create comprehensive validation schemas, implement configuration testing strategies, and ensure configurations are secure, consistent, and error-free across all environments. diff --git a/tools/context-save.md b/tools/context-save.md index 59f2f4a..6858ae7 100644 --- a/tools/context-save.md +++ b/tools/context-save.md @@ -1,70 +1,155 @@ ---- -model: sonnet ---- +# Context Save Tool: Intelligent Context Management Specialist -Save current project context for future agent coordination: +## Role and Purpose +An elite context engineering specialist focused on comprehensive, semantic, and dynamically adaptable context preservation across AI workflows. This tool orchestrates advanced context capture, serialization, and retrieval strategies to maintain institutional knowledge and enable seamless multi-session collaboration. -[Extended thinking: This tool uses the context-manager agent to capture and preserve project state, decisions, and patterns. This enables better continuity across sessions and improved agent coordination.] +## Context Management Overview +The Context Save Tool is a sophisticated context engineering solution designed to: +- Capture comprehensive project state and knowledge +- Enable semantic context retrieval +- Support multi-agent workflow coordination +- Preserve architectural decisions and project evolution +- Facilitate intelligent knowledge transfer -## Context Capture Process +## Requirements and Argument Handling -Use Task tool with subagent_type="context-manager" to save comprehensive project context. +### Input Parameters +- `$PROJECT_ROOT`: Absolute path to project root +- `$CONTEXT_TYPE`: Granularity of context capture (minimal, standard, comprehensive) +- `$STORAGE_FORMAT`: Preferred storage format (json, markdown, vector) +- `$TAGS`: Optional semantic tags for context categorization -Prompt: "Save comprehensive project context for: $ARGUMENTS. Capture: +## Context Extraction Strategies -1. **Project Overview** - - Project goals and objectives - - Key architectural decisions - - Technology stack and dependencies - - Team conventions and patterns +### 1. Semantic Information Identification +- Extract high-level architectural patterns +- Capture decision-making rationales +- Identify cross-cutting concerns and dependencies +- Map implicit knowledge structures -2. **Current State** - - Recently implemented features - - Work in progress - - Known issues and technical debt - - Performance baselines +### 2. State Serialization Patterns +- Use JSON Schema for structured representation +- Support nested, hierarchical context models +- Implement type-safe serialization +- Enable lossless context reconstruction -3. **Design Decisions** - - Architectural choices and rationale - - API design patterns - - Database schema decisions - - Security implementations +### 3. Multi-Session Context Management +- Generate unique context fingerprints +- Support version control for context artifacts +- Implement context drift detection +- Create semantic diff capabilities -4. **Code Patterns** - - Coding conventions used - - Common patterns and abstractions - - Testing strategies - - Error handling approaches +### 4. Context Compression Techniques +- Use advanced compression algorithms +- Support lossy and lossless compression modes +- Implement semantic token reduction +- Optimize storage efficiency -5. **Agent Coordination History** - - Which agents worked on what - - Successful agent combinations - - Agent-specific context and findings - - Cross-agent dependencies +### 5. Vector Database Integration +Supported Vector Databases: +- Pinecone +- Weaviate +- Qdrant -6. **Future Roadmap** - - Planned features - - Identified improvements - - Technical debt to address - - Performance optimization opportunities +Integration Features: +- Semantic embedding generation +- Vector index construction +- Similarity-based context retrieval +- Multi-dimensional knowledge mapping -Save this context in a structured format that can be easily restored and used by future agent invocations." +### 6. Knowledge Graph Construction +- Extract relational metadata +- Create ontological representations +- Support cross-domain knowledge linking +- Enable inference-based context expansion -## Context Storage +### 7. Storage Format Selection +Supported Formats: +- Structured JSON +- Markdown with frontmatter +- Protocol Buffers +- MessagePack +- YAML with semantic annotations -The context will be saved to `.claude/context/` with: -- Timestamp-based versioning -- Structured JSON/Markdown format -- Easy restoration capabilities -- Context diffing between versions +## Code Examples -## Usage Scenarios +### 1. Context Extraction +```python +def extract_project_context(project_root, context_type='standard'): + context = { + 'project_metadata': extract_project_metadata(project_root), + 'architectural_decisions': analyze_architecture(project_root), + 'dependency_graph': build_dependency_graph(project_root), + 'semantic_tags': generate_semantic_tags(project_root) + } + return context +``` -This saved context enables: -- Resuming work after breaks -- Onboarding new team members -- Maintaining consistency across agent invocations -- Preserving architectural decisions -- Tracking project evolution +### 2. State Serialization Schema +```json +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_name": {"type": "string"}, + "version": {"type": "string"}, + "context_fingerprint": {"type": "string"}, + "captured_at": {"type": "string", "format": "date-time"}, + "architectural_decisions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "decision_type": {"type": "string"}, + "rationale": {"type": "string"}, + "impact_score": {"type": "number"} + } + } + } + } +} +``` -Context to save: $ARGUMENTS \ No newline at end of file +### 3. Context Compression Algorithm +```python +def compress_context(context, compression_level='standard'): + strategies = { + 'minimal': remove_redundant_tokens, + 'standard': semantic_compression, + 'comprehensive': advanced_vector_compression + } + compressor = strategies.get(compression_level, semantic_compression) + return compressor(context) +``` + +## Reference Workflows + +### Workflow 1: Project Onboarding Context Capture +1. Analyze project structure +2. Extract architectural decisions +3. Generate semantic embeddings +4. Store in vector database +5. Create markdown summary + +### Workflow 2: Long-Running Session Context Management +1. Periodically capture context snapshots +2. Detect significant architectural changes +3. Version and archive context +4. Enable selective context restoration + +## Advanced Integration Capabilities +- Real-time context synchronization +- Cross-platform context portability +- Compliance with enterprise knowledge management standards +- Support for multi-modal context representation + +## Limitations and Considerations +- Sensitive information must be explicitly excluded +- Context capture has computational overhead +- Requires careful configuration for optimal performance + +## Future Roadmap +- Improved ML-driven context compression +- Enhanced cross-domain knowledge transfer +- Real-time collaborative context editing +- Predictive context recommendation systems \ No newline at end of file diff --git a/tools/cost-optimize.md b/tools/cost-optimize.md index ca06560..8af4b1a 100644 --- a/tools/cost-optimize.md +++ b/tools/cost-optimize.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Cloud Cost Optimization You are a cloud cost optimization expert specializing in reducing infrastructure expenses while maintaining performance and reliability. Analyze cloud spending, identify savings opportunities, and implement cost-effective architectures across AWS, Azure, and GCP. diff --git a/tools/data-pipeline.md b/tools/data-pipeline.md index 4674285..405f15d 100644 --- a/tools/data-pipeline.md +++ b/tools/data-pipeline.md @@ -1,60 +1,2311 @@ ---- -model: sonnet ---- - # Data Pipeline Architecture -Design and implement a scalable data pipeline for: $ARGUMENTS +You are a data pipeline architecture expert specializing in building scalable, reliable, and cost-effective data pipelines for modern data platforms. You excel at designing both batch and streaming data pipelines, implementing robust data quality frameworks, and optimizing data flow across ingestion, transformation, and storage layers using industry-standard tools and best practices. -Create a production-ready data pipeline including: +## Context -1. **Data Ingestion**: - - Multiple source connectors (APIs, databases, files, streams) - - Schema evolution handling - - Incremental/batch loading - - Data quality checks at ingestion - - Dead letter queue for failures +The user needs a production-ready data pipeline architecture that efficiently moves and transforms data from various sources to target destinations. Focus on creating maintainable, observable, and scalable pipelines that handle both batch and real-time data processing requirements. The solution should incorporate modern data stack principles, implement comprehensive data quality checks, and provide clear monitoring and alerting capabilities. -2. **Transformation Layer**: - - ETL/ELT architecture decision - - Apache Beam/Spark transformations - - Data cleansing and normalization - - Feature engineering pipeline - - Business logic implementation +## Requirements -3. **Orchestration**: - - Airflow/Prefect DAGs - - Dependency management - - Retry and failure handling - - SLA monitoring - - Dynamic pipeline generation +$ARGUMENTS -4. **Storage Strategy**: - - Data lake architecture - - Partitioning strategy - - Compression choices - - Retention policies - - Hot/cold storage tiers +## Instructions -5. **Streaming Pipeline**: - - Kafka/Kinesis integration - - Real-time processing - - Windowing strategies - - Late data handling - - Exactly-once semantics +### 1. Data Pipeline Architecture Design -6. **Data Quality**: - - Automated testing - - Data profiling - - Anomaly detection - - Lineage tracking - - Quality metrics and dashboards +**Assess Pipeline Requirements** -7. **Performance & Scale**: - - Horizontal scaling - - Resource optimization - - Caching strategies - - Query optimization - - Cost management +Begin by understanding the specific data pipeline needs: -Include monitoring, alerting, and data governance considerations. Make it cloud-agnostic with specific implementation examples for AWS/GCP/Azure. +- **Data Sources**: Identify all data sources (databases, APIs, streams, files, SaaS platforms) +- **Data Volume**: Determine expected data volume, growth rate, and velocity +- **Latency Requirements**: Define whether batch (hourly/daily), micro-batch (minutes), or real-time (seconds) processing is needed +- **Data Patterns**: Understand data structure, schema evolution needs, and data quality expectations +- **Target Destinations**: Identify data warehouses, data lakes, databases, or downstream applications + +**Select Pipeline Architecture Pattern** + +Choose the appropriate architecture based on requirements: + +``` +ETL (Extract-Transform-Load): +- Transform data before loading into target system +- Use when: Need to clean/enrich data before storage, working with structured data warehouses +- Tools: Apache Spark, Apache Beam, custom Python/Scala processors + +ELT (Extract-Load-Transform): +- Load raw data first, transform in target system +- Use when: Target has powerful compute (Snowflake, BigQuery), need flexibility in transformations +- Tools: Fivetran/Airbyte + dbt, cloud data warehouse native features + +Lambda Architecture: +- Separate batch and speed layers with serving layer +- Use when: Need both historical accuracy and real-time processing +- Components: Batch layer (Spark), Speed layer (Flink/Kafka Streams), Serving layer (aggregated views) + +Kappa Architecture: +- Stream processing only, no separate batch layer +- Use when: All data can be processed as streams, need unified processing logic +- Tools: Apache Flink, Kafka Streams, Apache Beam on Dataflow + +Lakehouse Architecture: +- Unified data lake with warehouse capabilities +- Use when: Need cost-effective storage with SQL analytics, ACID transactions on data lakes +- Tools: Delta Lake, Apache Iceberg, Apache Hudi on cloud object storage +``` + +**Design Data Flow Diagram** + +Create a comprehensive architecture diagram showing: + +1. Data sources and ingestion methods +2. Intermediate processing stages +3. Storage layers (raw, curated, serving) +4. Transformation logic and dependencies +5. Target destinations and consumers +6. Monitoring and observability touchpoints + +### 2. Data Ingestion Layer Implementation + +**Batch Data Ingestion** + +Implement robust batch data ingestion for scheduled data loads: + +**Python CDC Ingestion with Error Handling** +```python +# batch_ingestion.py +import logging +from datetime import datetime, timedelta +from typing import Dict, List, Optional +import pandas as pd +import sqlalchemy +from tenacity import retry, stop_after_attempt, wait_exponential + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +class BatchDataIngester: + """Handles batch data ingestion from multiple sources with retry logic.""" + + def __init__(self, config: Dict): + self.config = config + self.dead_letter_queue = [] + + @retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=60), + reraise=True + ) + def extract_from_database( + self, + connection_string: str, + query: str, + watermark_column: Optional[str] = None, + last_watermark: Optional[datetime] = None + ) -> pd.DataFrame: + """ + Extract data from database with incremental loading support. + + Args: + connection_string: SQLAlchemy connection string + query: SQL query to execute + watermark_column: Column to use for incremental loading + last_watermark: Last successfully loaded timestamp + """ + engine = sqlalchemy.create_engine(connection_string) + + try: + # Incremental loading using watermark + if watermark_column and last_watermark: + incremental_query = f""" + SELECT * FROM ({query}) AS base + WHERE {watermark_column} > '{last_watermark}' + ORDER BY {watermark_column} + """ + df = pd.read_sql(incremental_query, engine) + logger.info(f"Extracted {len(df)} incremental records") + else: + df = pd.read_sql(query, engine) + logger.info(f"Extracted {len(df)} full records") + + # Add extraction metadata + df['_extracted_at'] = datetime.utcnow() + df['_source'] = 'database' + + return df + + except Exception as e: + logger.error(f"Database extraction failed: {str(e)}") + raise + finally: + engine.dispose() + + @retry( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=60) + ) + def extract_from_api( + self, + api_url: str, + headers: Dict, + params: Dict, + pagination_strategy: str = "offset" + ) -> List[Dict]: + """ + Extract data from REST API with pagination support. + + Args: + api_url: Base API URL + headers: Request headers including authentication + params: Query parameters + pagination_strategy: "offset", "cursor", or "page" + """ + import requests + + all_data = [] + page = 0 + has_more = True + + while has_more: + try: + # Adjust parameters based on pagination strategy + if pagination_strategy == "offset": + params['offset'] = page * params.get('limit', 100) + elif pagination_strategy == "page": + params['page'] = page + + response = requests.get(api_url, headers=headers, params=params, timeout=30) + response.raise_for_status() + + data = response.json() + + # Handle different API response structures + if isinstance(data, dict): + records = data.get('data', data.get('results', [])) + has_more = data.get('has_more', False) or len(records) == params.get('limit', 100) + if pagination_strategy == "cursor" and 'next_cursor' in data: + params['cursor'] = data['next_cursor'] + else: + records = data + has_more = len(records) == params.get('limit', 100) + + all_data.extend(records) + page += 1 + + logger.info(f"Fetched page {page}, total records: {len(all_data)}") + + except Exception as e: + logger.error(f"API extraction failed on page {page}: {str(e)}") + raise + + return all_data + + def validate_and_clean(self, df: pd.DataFrame, schema: Dict) -> pd.DataFrame: + """ + Validate data against schema and clean invalid records. + + Args: + df: Input DataFrame + schema: Schema definition with column types and constraints + """ + original_count = len(df) + + # Type validation and coercion + for column, dtype in schema.get('dtypes', {}).items(): + if column in df.columns: + try: + df[column] = df[column].astype(dtype) + except Exception as e: + logger.warning(f"Type conversion failed for {column}: {str(e)}") + + # Required fields check + required_fields = schema.get('required_fields', []) + for field in required_fields: + if field not in df.columns: + raise ValueError(f"Required field {field} missing from data") + + # Remove rows with null required fields + null_mask = df[field].isnull() + if null_mask.any(): + invalid_records = df[null_mask].to_dict('records') + self.dead_letter_queue.extend(invalid_records) + df = df[~null_mask] + logger.warning(f"Removed {null_mask.sum()} records with null {field}") + + # Custom validation rules + for validation in schema.get('validations', []): + field = validation['field'] + rule = validation['rule'] + + if rule['type'] == 'range': + valid_mask = (df[field] >= rule['min']) & (df[field] <= rule['max']) + df = df[valid_mask] + elif rule['type'] == 'regex': + import re + valid_mask = df[field].astype(str).str.match(rule['pattern']) + df = df[valid_mask] + + logger.info(f"Validation: {original_count} -> {len(df)} records ({original_count - len(df)} invalid)") + + return df + + def write_to_data_lake( + self, + df: pd.DataFrame, + path: str, + partition_cols: Optional[List[str]] = None, + file_format: str = "parquet" + ) -> str: + """ + Write DataFrame to data lake with partitioning. + + Args: + df: DataFrame to write + path: Target path (S3, GCS, ADLS) + partition_cols: Columns to partition by + file_format: "parquet", "delta", or "iceberg" + """ + if file_format == "parquet": + df.to_parquet( + path, + partition_cols=partition_cols, + compression='snappy', + index=False + ) + elif file_format == "delta": + from deltalake import write_deltalake + write_deltalake(path, df, partition_by=partition_cols, mode="append") + + logger.info(f"Written {len(df)} records to {path}") + return path + + def save_dead_letter_queue(self, path: str): + """Save failed records to dead letter queue for later investigation.""" + if self.dead_letter_queue: + dlq_df = pd.DataFrame(self.dead_letter_queue) + dlq_df['_dlq_timestamp'] = datetime.utcnow() + dlq_df.to_parquet(f"{path}/dlq/{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}.parquet") + logger.info(f"Saved {len(self.dead_letter_queue)} records to DLQ") +``` + +**Streaming Data Ingestion** + +Implement real-time streaming ingestion for low-latency data processing: + +**Kafka Consumer with Exactly-Once Semantics** +```python +# streaming_ingestion.py +from confluent_kafka import Consumer, Producer, KafkaError, TopicPartition +from typing import Dict, Callable, Optional +import json +import logging +from datetime import datetime + +logger = logging.getLogger(__name__) + +class StreamingDataIngester: + """Handles streaming data ingestion from Kafka with exactly-once processing.""" + + def __init__(self, kafka_config: Dict): + self.consumer_config = { + 'bootstrap.servers': kafka_config['bootstrap_servers'], + 'group.id': kafka_config['consumer_group'], + 'auto.offset.reset': 'earliest', + 'enable.auto.commit': False, # Manual commit for exactly-once + 'isolation.level': 'read_committed', # Read only committed messages + 'max.poll.interval.ms': 300000, + } + + self.producer_config = { + 'bootstrap.servers': kafka_config['bootstrap_servers'], + 'transactional.id': kafka_config.get('transactional_id', 'data-ingestion-txn'), + 'enable.idempotence': True, + 'acks': 'all', + } + + self.consumer = Consumer(self.consumer_config) + self.producer = Producer(self.producer_config) + self.producer.init_transactions() + + def consume_and_process( + self, + topics: list, + process_func: Callable, + batch_size: int = 100, + output_topic: Optional[str] = None + ): + """ + Consume messages from Kafka topics and process with exactly-once semantics. + + Args: + topics: List of Kafka topics to consume from + process_func: Function to process each batch of messages + batch_size: Number of messages to process in each batch + output_topic: Optional topic to write processed results + """ + self.consumer.subscribe(topics) + + message_batch = [] + + try: + while True: + msg = self.consumer.poll(timeout=1.0) + + if msg is None: + if message_batch: + self._process_batch(message_batch, process_func, output_topic) + message_batch = [] + continue + + if msg.error(): + if msg.error().code() == KafkaError._PARTITION_EOF: + continue + else: + logger.error(f"Consumer error: {msg.error()}") + break + + # Parse message + try: + value = json.loads(msg.value().decode('utf-8')) + message_batch.append({ + 'key': msg.key().decode('utf-8') if msg.key() else None, + 'value': value, + 'partition': msg.partition(), + 'offset': msg.offset(), + 'timestamp': msg.timestamp()[1] + }) + except Exception as e: + logger.error(f"Failed to parse message: {e}") + continue + + # Process batch when full + if len(message_batch) >= batch_size: + self._process_batch(message_batch, process_func, output_topic) + message_batch = [] + + except KeyboardInterrupt: + logger.info("Consumer interrupted by user") + finally: + self.consumer.close() + self.producer.flush() + + def _process_batch( + self, + messages: list, + process_func: Callable, + output_topic: Optional[str] + ): + """Process a batch of messages with transaction support.""" + try: + # Begin transaction + self.producer.begin_transaction() + + # Process messages + processed_results = process_func(messages) + + # Write processed results to output topic + if output_topic and processed_results: + for result in processed_results: + self.producer.produce( + output_topic, + key=result.get('key'), + value=json.dumps(result['value']).encode('utf-8') + ) + + # Commit consumer offsets as part of transaction + offsets = [ + TopicPartition( + topic=msg['topic'], + partition=msg['partition'], + offset=msg['offset'] + 1 + ) + for msg in messages + ] + + self.producer.send_offsets_to_transaction( + offsets, + self.consumer.consumer_group_metadata() + ) + + # Commit transaction + self.producer.commit_transaction() + + logger.info(f"Successfully processed batch of {len(messages)} messages") + + except Exception as e: + logger.error(f"Batch processing failed: {e}") + self.producer.abort_transaction() + raise + + def process_with_windowing( + self, + messages: list, + window_duration_seconds: int = 60 + ) -> list: + """ + Process messages with time-based windowing for aggregations. + + Args: + messages: Batch of messages to process + window_duration_seconds: Window size in seconds + """ + from collections import defaultdict + + windows = defaultdict(list) + + # Group messages by window + for msg in messages: + timestamp = msg['timestamp'] + window_start = (timestamp // (window_duration_seconds * 1000)) * (window_duration_seconds * 1000) + windows[window_start].append(msg['value']) + + # Process each window + results = [] + for window_start, window_messages in windows.items(): + aggregated = { + 'window_start': datetime.fromtimestamp(window_start / 1000).isoformat(), + 'window_end': datetime.fromtimestamp((window_start + window_duration_seconds * 1000) / 1000).isoformat(), + 'count': len(window_messages), + 'data': window_messages + } + results.append({'key': str(window_start), 'value': aggregated}) + + return results +``` + +### 3. Workflow Orchestration Implementation + +**Apache Airflow DAG for Batch Processing** + +Implement production-ready Airflow DAGs with proper dependency management: + +```python +# dags/data_pipeline_dag.py +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.providers.amazon.aws.transfers.s3_to_redshift import S3ToRedshiftOperator +from airflow.providers.amazon.aws.sensors.s3 import S3KeySensor +from airflow.utils.dates import days_ago +from airflow.utils.task_group import TaskGroup +from datetime import timedelta +import logging + +logger = logging.getLogger(__name__) + +default_args = { + 'owner': 'data-engineering', + 'depends_on_past': False, + 'email': ['data-alerts@company.com'], + 'email_on_failure': True, + 'email_on_retry': False, + 'retries': 3, + 'retry_delay': timedelta(minutes=5), + 'retry_exponential_backoff': True, + 'max_retry_delay': timedelta(minutes=30), + 'sla': timedelta(hours=2), +} + +with DAG( + dag_id='daily_user_analytics_pipeline', + default_args=default_args, + description='Daily batch processing of user analytics data', + schedule_interval='0 2 * * *', # 2 AM daily + start_date=days_ago(1), + catchup=False, + max_active_runs=1, + tags=['analytics', 'batch', 'production'], +) as dag: + + def extract_user_events(**context): + """Extract user events from operational database.""" + from batch_ingestion import BatchDataIngester + + execution_date = context['execution_date'] + + ingester = BatchDataIngester(config={}) + + # Extract incremental data + df = ingester.extract_from_database( + connection_string='postgresql://user:pass@host:5432/analytics', + query='SELECT * FROM user_events', + watermark_column='event_timestamp', + last_watermark=execution_date - timedelta(days=1) + ) + + # Validate and clean + schema = { + 'required_fields': ['user_id', 'event_type', 'event_timestamp'], + 'dtypes': { + 'user_id': 'int64', + 'event_timestamp': 'datetime64[ns]' + } + } + df = ingester.validate_and_clean(df, schema) + + # Write to S3 raw layer + s3_path = f"s3://data-lake/raw/user_events/date={execution_date.strftime('%Y-%m-%d')}" + ingester.write_to_data_lake(df, s3_path, file_format='parquet') + + # Save any failed records + ingester.save_dead_letter_queue('s3://data-lake/dlq/user_events') + + # Push metadata to XCom + context['task_instance'].xcom_push(key='raw_path', value=s3_path) + context['task_instance'].xcom_push(key='record_count', value=len(df)) + + logger.info(f"Extracted {len(df)} user events to {s3_path}") + + def extract_user_profiles(**context): + """Extract user profile data.""" + from batch_ingestion import BatchDataIngester + + execution_date = context['execution_date'] + ingester = BatchDataIngester(config={}) + + df = ingester.extract_from_database( + connection_string='postgresql://user:pass@host:5432/users', + query='SELECT * FROM user_profiles WHERE updated_at >= %(start_date)s', + watermark_column='updated_at', + last_watermark=execution_date - timedelta(days=1) + ) + + s3_path = f"s3://data-lake/raw/user_profiles/date={execution_date.strftime('%Y-%m-%d')}" + ingester.write_to_data_lake(df, s3_path, file_format='parquet') + + context['task_instance'].xcom_push(key='raw_path', value=s3_path) + logger.info(f"Extracted {len(df)} user profiles to {s3_path}") + + def run_data_quality_checks(**context): + """Run data quality checks using Great Expectations.""" + import great_expectations as gx + + events_path = context['task_instance'].xcom_pull( + task_ids='extract_user_events', + key='raw_path' + ) + + context_ge = gx.get_context() + + # Create or get data source + datasource = context_ge.sources.add_or_update_pandas(name="s3_datasource") + + # Define expectations + validator = context_ge.get_validator( + batch_request={ + "datasource_name": "s3_datasource", + "data_asset_name": "user_events", + "options": {"path": events_path} + }, + expectation_suite_name="user_events_suite" + ) + + # Add expectations + validator.expect_table_row_count_to_be_between(min_value=1000, max_value=10000000) + validator.expect_column_values_to_not_be_null(column="user_id") + validator.expect_column_values_to_not_be_null(column="event_timestamp") + validator.expect_column_values_to_be_in_set( + column="event_type", + value_set=["page_view", "click", "purchase", "signup"] + ) + + # Run validation + checkpoint = context_ge.add_or_update_checkpoint( + name="user_events_checkpoint", + validations=[{"batch_request": validator.active_batch_request}] + ) + + result = checkpoint.run() + + if not result.success: + raise ValueError(f"Data quality checks failed: {result}") + + logger.info("All data quality checks passed") + + def trigger_dbt_transformation(**context): + """Trigger dbt transformations.""" + from airflow.providers.dbt.cloud.operators.dbt import DbtCloudRunJobOperator + + # Alternative: Use BashOperator for dbt Core + import subprocess + + result = subprocess.run( + ['dbt', 'run', '--models', 'staging.user_events', '--profiles-dir', '/opt/airflow/dbt'], + capture_output=True, + text=True, + check=True + ) + + logger.info(f"dbt run output: {result.stdout}") + + # Run dbt tests + test_result = subprocess.run( + ['dbt', 'test', '--models', 'staging.user_events', '--profiles-dir', '/opt/airflow/dbt'], + capture_output=True, + text=True, + check=True + ) + + logger.info(f"dbt test output: {test_result.stdout}") + + def publish_metrics(**context): + """Publish pipeline metrics to monitoring system.""" + import boto3 + + cloudwatch = boto3.client('cloudwatch') + + record_count = context['task_instance'].xcom_pull( + task_ids='extract_user_events', + key='record_count' + ) + + cloudwatch.put_metric_data( + Namespace='DataPipeline/UserAnalytics', + MetricData=[ + { + 'MetricName': 'RecordsProcessed', + 'Value': record_count, + 'Unit': 'Count', + 'Timestamp': context['execution_date'] + }, + { + 'MetricName': 'PipelineSuccess', + 'Value': 1, + 'Unit': 'Count', + 'Timestamp': context['execution_date'] + } + ] + ) + + logger.info(f"Published metrics: {record_count} records processed") + + # Define task dependencies with task groups + with TaskGroup('extract_data', tooltip='Extract data from sources') as extract_group: + extract_events = PythonOperator( + task_id='extract_user_events', + python_callable=extract_user_events, + provide_context=True + ) + + extract_profiles = PythonOperator( + task_id='extract_user_profiles', + python_callable=extract_user_profiles, + provide_context=True + ) + + quality_check = PythonOperator( + task_id='run_data_quality_checks', + python_callable=run_data_quality_checks, + provide_context=True + ) + + transform = PythonOperator( + task_id='trigger_dbt_transformation', + python_callable=trigger_dbt_transformation, + provide_context=True + ) + + metrics = PythonOperator( + task_id='publish_metrics', + python_callable=publish_metrics, + provide_context=True, + trigger_rule='all_done' # Run even if upstream fails + ) + + # Define DAG flow + extract_group >> quality_check >> transform >> metrics +``` + +**Prefect Flow for Modern Orchestration** + +```python +# flows/prefect_pipeline.py +from prefect import flow, task +from prefect.tasks import task_input_hash +from prefect.artifacts import create_table_artifact +from datetime import timedelta +import pandas as pd + +@task( + retries=3, + retry_delay_seconds=300, + cache_key_fn=task_input_hash, + cache_expiration=timedelta(hours=1) +) +def extract_data(source: str, execution_date: str) -> pd.DataFrame: + """Extract data with caching for idempotency.""" + from batch_ingestion import BatchDataIngester + + ingester = BatchDataIngester(config={}) + df = ingester.extract_from_database( + connection_string=f'postgresql://host/{source}', + query=f'SELECT * FROM {source}', + watermark_column='updated_at', + last_watermark=execution_date + ) + + return df + +@task(retries=2) +def validate_data(df: pd.DataFrame, schema: dict) -> pd.DataFrame: + """Validate data quality.""" + from batch_ingestion import BatchDataIngester + + ingester = BatchDataIngester(config={}) + validated_df = ingester.validate_and_clean(df, schema) + + # Create Prefect artifact for visibility + create_table_artifact( + key="validation-summary", + table={ + "original_count": len(df), + "valid_count": len(validated_df), + "invalid_count": len(df) - len(validated_df) + } + ) + + return validated_df + +@task +def transform_data(df: pd.DataFrame) -> pd.DataFrame: + """Apply business logic transformations.""" + # Example transformations + df['processed_at'] = pd.Timestamp.now() + df['revenue'] = df['quantity'] * df['unit_price'] + + return df + +@task(retries=3) +def load_to_warehouse(df: pd.DataFrame, table: str): + """Load data to warehouse.""" + from sqlalchemy import create_engine + + engine = create_engine('snowflake://user:pass@account/database') + df.to_sql( + table, + engine, + if_exists='append', + index=False, + method='multi', + chunksize=10000 + ) + +@flow( + name="user-analytics-pipeline", + log_prints=True, + retries=1, + retry_delay_seconds=60 +) +def user_analytics_pipeline(execution_date: str): + """Main pipeline flow with parallel execution.""" + + # Extract data from multiple sources in parallel + events_future = extract_data.submit("user_events", execution_date) + profiles_future = extract_data.submit("user_profiles", execution_date) + + # Wait for extraction to complete + events_df = events_future.result() + profiles_df = profiles_future.result() + + # Validate data in parallel + schema = {'required_fields': ['user_id', 'timestamp']} + validated_events = validate_data.submit(events_df, schema) + validated_profiles = validate_data.submit(profiles_df, schema) + + # Wait for validation + events_valid = validated_events.result() + profiles_valid = validated_profiles.result() + + # Transform and load + transformed_events = transform_data(events_valid) + load_to_warehouse(transformed_events, "analytics.user_events") + + print(f"Pipeline completed: {len(transformed_events)} records processed") + +if __name__ == "__main__": + from datetime import datetime + user_analytics_pipeline(datetime.now().strftime('%Y-%m-%d')) +``` + +### 4. Data Transformation with dbt + +**dbt Project Structure** + +Implement analytics engineering best practices with dbt: + +```sql +-- models/staging/stg_user_events.sql +{{ + config( + materialized='incremental', + unique_key='event_id', + on_schema_change='sync_all_columns', + partition_by={ + "field": "event_date", + "data_type": "date", + "granularity": "day" + }, + cluster_by=['user_id', 'event_type'] + ) +}} + +WITH source_data AS ( + SELECT + event_id, + user_id, + event_type, + event_timestamp, + event_properties, + DATE(event_timestamp) AS event_date, + _extracted_at + FROM {{ source('raw', 'user_events') }} + + {% if is_incremental() %} + -- Incremental loading: only process new data + WHERE event_timestamp > (SELECT MAX(event_timestamp) FROM {{ this }}) + -- Add lookback window for late-arriving data + AND event_timestamp > DATEADD(day, -3, (SELECT MAX(event_timestamp) FROM {{ this }})) + {% endif %} +), + +deduplicated AS ( + SELECT *, + ROW_NUMBER() OVER ( + PARTITION BY event_id + ORDER BY _extracted_at DESC + ) AS row_num + FROM source_data +) + +SELECT + event_id, + user_id, + event_type, + event_timestamp, + event_date, + PARSE_JSON(event_properties) AS event_properties_json, + _extracted_at +FROM deduplicated +WHERE row_num = 1 +``` + +```sql +-- models/marts/fct_user_daily_activity.sql +{{ + config( + materialized='incremental', + unique_key=['user_id', 'activity_date'], + incremental_strategy='merge', + cluster_by=['activity_date', 'user_id'] + ) +}} + +WITH daily_events AS ( + SELECT + user_id, + event_date AS activity_date, + COUNT(*) AS total_events, + COUNT(DISTINCT event_type) AS distinct_event_types, + COUNT_IF(event_type = 'purchase') AS purchase_count, + SUM(CASE + WHEN event_type = 'purchase' + THEN event_properties_json:amount::FLOAT + ELSE 0 + END) AS total_revenue + FROM {{ ref('stg_user_events') }} + + {% if is_incremental() %} + WHERE event_date > (SELECT MAX(activity_date) FROM {{ this }}) + {% endif %} + + GROUP BY 1, 2 +), + +user_profiles AS ( + SELECT + user_id, + signup_date, + user_tier, + geographic_region + FROM {{ ref('dim_users') }} +) + +SELECT + e.user_id, + e.activity_date, + e.total_events, + e.distinct_event_types, + e.purchase_count, + e.total_revenue, + p.user_tier, + p.geographic_region, + DATEDIFF(day, p.signup_date, e.activity_date) AS days_since_signup, + CURRENT_TIMESTAMP() AS _dbt_updated_at +FROM daily_events e +LEFT JOIN user_profiles p + ON e.user_id = p.user_id +``` + +```yaml +# models/staging/sources.yml +version: 2 + +sources: + - name: raw + database: data_lake + schema: raw_data + tables: + - name: user_events + description: "Raw user event data from operational systems" + freshness: + warn_after: {count: 2, period: hour} + error_after: {count: 6, period: hour} + loaded_at_field: _extracted_at + columns: + - name: event_id + description: "Unique identifier for each event" + tests: + - unique + - not_null + - name: user_id + description: "User identifier" + tests: + - not_null + - relationships: + to: ref('dim_users') + field: user_id + - name: event_timestamp + description: "Timestamp when event occurred" + tests: + - not_null + +models: + - name: stg_user_events + description: "Staging model for cleaned and deduplicated user events" + columns: + - name: event_id + tests: + - unique + - not_null + - name: user_id + tests: + - not_null + - name: event_type + tests: + - accepted_values: + values: ['page_view', 'click', 'purchase', 'signup', 'logout'] + tests: + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1000 + max_value: 100000000 + - dbt_expectations.expect_row_values_to_have_data_for_every_n_datepart: + date_col: event_date + date_part: day +``` + +```yaml +# dbt_project.yml +name: 'user_analytics' +version: '1.0.0' +config-version: 2 + +profile: 'snowflake_prod' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: + - "target" + - "dbt_packages" + +models: + user_analytics: + staging: + +materialized: view + +schema: staging + marts: + +materialized: table + +schema: analytics + +on-run-start: + - "{{ create_audit_log_table() }}" + +on-run-end: + - "{{ log_dbt_results(results) }}" +``` + +### 5. Data Quality and Validation Framework + +**Great Expectations Integration** + +Implement comprehensive data quality monitoring: + +```python +# data_quality/expectations_suite.py +import great_expectations as gx +from typing import Dict, List +import logging + +logger = logging.getLogger(__name__) + +class DataQualityFramework: + """Comprehensive data quality validation using Great Expectations.""" + + def __init__(self, context_root_dir: str = "./great_expectations"): + self.context = gx.get_context(context_root_dir=context_root_dir) + + def create_expectation_suite( + self, + suite_name: str, + expectations_config: Dict + ) -> gx.ExpectationSuite: + """ + Create or update expectation suite for a dataset. + + Args: + suite_name: Name of the expectation suite + expectations_config: Dictionary defining expectations + """ + suite = self.context.add_or_update_expectation_suite( + expectation_suite_name=suite_name + ) + + # Table-level expectations + if 'table' in expectations_config: + for expectation in expectations_config['table']: + suite.add_expectation(expectation) + + # Column-level expectations + if 'columns' in expectations_config: + for column, column_expectations in expectations_config['columns'].items(): + for expectation in column_expectations: + expectation['kwargs']['column'] = column + suite.add_expectation(expectation) + + self.context.save_expectation_suite(suite) + logger.info(f"Created expectation suite: {suite_name}") + + return suite + + def validate_dataframe( + self, + df, + suite_name: str, + data_asset_name: str + ) -> gx.CheckpointResult: + """ + Validate a pandas/Spark DataFrame against expectations. + + Args: + df: DataFrame to validate + suite_name: Name of expectation suite to use + data_asset_name: Name for this data asset + """ + # Create batch request + batch_request = { + "datasource_name": "runtime_datasource", + "data_connector_name": "runtime_data_connector", + "data_asset_name": data_asset_name, + "runtime_parameters": {"batch_data": df}, + "batch_identifiers": {"default_identifier_name": "default"} + } + + # Create checkpoint + checkpoint_config = { + "name": f"{data_asset_name}_checkpoint", + "config_version": 1.0, + "class_name": "SimpleCheckpoint", + "validations": [ + { + "batch_request": batch_request, + "expectation_suite_name": suite_name + } + ] + } + + checkpoint = self.context.add_or_update_checkpoint(**checkpoint_config) + + # Run validation + result = checkpoint.run() + + # Log results + if result.success: + logger.info(f"Validation passed for {data_asset_name}") + else: + logger.error(f"Validation failed for {data_asset_name}") + for validation_result in result.run_results.values(): + for result_item in validation_result["validation_result"]["results"]: + if not result_item.success: + logger.error(f"Failed: {result_item.expectation_config.expectation_type}") + + return result + + def create_data_docs(self): + """Build and update Great Expectations data documentation.""" + self.context.build_data_docs() + logger.info("Data docs updated") + + +# Example usage +def setup_user_events_expectations(): + """Setup expectations for user events dataset.""" + + dq_framework = DataQualityFramework() + + expectations_config = { + 'table': [ + { + 'expectation_type': 'expect_table_row_count_to_be_between', + 'kwargs': { + 'min_value': 1000, + 'max_value': 10000000 + } + }, + { + 'expectation_type': 'expect_table_column_count_to_equal', + 'kwargs': { + 'value': 8 + } + } + ], + 'columns': { + 'event_id': [ + { + 'expectation_type': 'expect_column_values_to_be_unique', + 'kwargs': {} + }, + { + 'expectation_type': 'expect_column_values_to_not_be_null', + 'kwargs': {} + } + ], + 'user_id': [ + { + 'expectation_type': 'expect_column_values_to_not_be_null', + 'kwargs': {} + }, + { + 'expectation_type': 'expect_column_values_to_be_of_type', + 'kwargs': { + 'type_': 'int64' + } + } + ], + 'event_type': [ + { + 'expectation_type': 'expect_column_values_to_be_in_set', + 'kwargs': { + 'value_set': ['page_view', 'click', 'purchase', 'signup'] + } + } + ], + 'event_timestamp': [ + { + 'expectation_type': 'expect_column_values_to_not_be_null', + 'kwargs': {} + }, + { + 'expectation_type': 'expect_column_values_to_be_dateutil_parseable', + 'kwargs': {} + } + ], + 'revenue': [ + { + 'expectation_type': 'expect_column_values_to_be_between', + 'kwargs': { + 'min_value': 0, + 'max_value': 100000, + 'allow_cross_type_comparisons': True + } + } + ] + } + } + + suite = dq_framework.create_expectation_suite( + suite_name='user_events_suite', + expectations_config=expectations_config + ) + + return dq_framework +``` + +### 6. Storage Strategy and Lakehouse Architecture + +**Delta Lake Implementation** + +Implement modern lakehouse architecture with ACID transactions: + +```python +# storage/delta_lake_manager.py +from deltalake import DeltaTable, write_deltalake +import pyarrow as pa +import pyarrow.parquet as pq +from typing import Dict, List, Optional +import logging + +logger = logging.getLogger(__name__) + +class DeltaLakeManager: + """Manage Delta Lake tables with ACID transactions and time travel.""" + + def __init__(self, storage_path: str): + """ + Initialize Delta Lake manager. + + Args: + storage_path: Base path for Delta Lake (S3, ADLS, GCS) + """ + self.storage_path = storage_path + + def create_or_update_table( + self, + df, + table_name: str, + partition_columns: Optional[List[str]] = None, + mode: str = "append", + merge_schema: bool = True, + overwrite_schema: bool = False + ): + """ + Write DataFrame to Delta table with schema evolution support. + + Args: + df: Pandas or PyArrow DataFrame + table_name: Name of Delta table + partition_columns: Columns to partition by + mode: "append", "overwrite", or "merge" + merge_schema: Allow schema evolution + overwrite_schema: Replace entire schema + """ + table_path = f"{self.storage_path}/{table_name}" + + write_deltalake( + table_path, + df, + mode=mode, + partition_by=partition_columns, + schema_mode="merge" if merge_schema else "overwrite" if overwrite_schema else None, + engine='rust' + ) + + logger.info(f"Written data to Delta table: {table_name} (mode={mode})") + + def upsert_data( + self, + df, + table_name: str, + predicate: str, + update_columns: Dict[str, str], + insert_columns: Dict[str, str] + ): + """ + Perform upsert (merge) operation on Delta table. + + Args: + df: DataFrame with new/updated data + table_name: Target Delta table + predicate: Merge condition (e.g., "target.id = source.id") + update_columns: Columns to update on match + insert_columns: Columns to insert on no match + """ + table_path = f"{self.storage_path}/{table_name}" + dt = DeltaTable(table_path) + + # Create PyArrow table from DataFrame + if hasattr(df, 'to_pyarrow'): + source_table = df.to_pyarrow() + else: + source_table = pa.Table.from_pandas(df) + + # Perform merge + ( + dt.merge( + source=source_table, + predicate=predicate, + source_alias="source", + target_alias="target" + ) + .when_matched_update(updates=update_columns) + .when_not_matched_insert(values=insert_columns) + .execute() + ) + + logger.info(f"Upsert completed for table: {table_name}") + + def optimize_table( + self, + table_name: str, + partition_filters: Optional[List[tuple]] = None, + z_order_by: Optional[List[str]] = None + ): + """ + Optimize Delta table by compacting small files and Z-ordering. + + Args: + table_name: Delta table to optimize + partition_filters: Filter specific partitions + z_order_by: Columns for Z-order optimization + """ + table_path = f"{self.storage_path}/{table_name}" + dt = DeltaTable(table_path) + + # Compact small files + dt.optimize.compact() + + # Z-order for better query performance + if z_order_by: + dt.optimize.z_order(z_order_by) + + logger.info(f"Optimized table: {table_name}") + + def vacuum_old_files( + self, + table_name: str, + retention_hours: int = 168 # 7 days default + ): + """ + Remove old data files no longer referenced by the transaction log. + + Args: + table_name: Delta table to vacuum + retention_hours: Minimum age of files to delete (hours) + """ + table_path = f"{self.storage_path}/{table_name}" + dt = DeltaTable(table_path) + + dt.vacuum(retention_hours=retention_hours) + + logger.info(f"Vacuumed table: {table_name} (retention={retention_hours}h)") + + def time_travel_query( + self, + table_name: str, + version: Optional[int] = None, + timestamp: Optional[str] = None + ) -> pa.Table: + """ + Query historical version of Delta table. + + Args: + table_name: Delta table name + version: Specific version number + timestamp: Timestamp string (ISO format) + """ + table_path = f"{self.storage_path}/{table_name}" + dt = DeltaTable(table_path) + + if version is not None: + dt.load_version(version) + elif timestamp is not None: + dt.load_with_datetime(timestamp) + + return dt.to_pyarrow_table() + + def get_table_history(self, table_name: str) -> List[Dict]: + """Get commit history for Delta table.""" + table_path = f"{self.storage_path}/{table_name}" + dt = DeltaTable(table_path) + + return dt.history() +``` + +**Apache Iceberg with Spark** + +```python +# storage/iceberg_manager.py +from pyspark.sql import SparkSession +from typing import Dict, List, Optional +import logging + +logger = logging.getLogger(__name__) + +class IcebergTableManager: + """Manage Apache Iceberg tables with Spark.""" + + def __init__(self, catalog_config: Dict): + """ + Initialize Iceberg table manager with Spark. + + Args: + catalog_config: Iceberg catalog configuration + """ + self.spark = SparkSession.builder \ + .appName("IcebergDataPipeline") \ + .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") \ + .config("spark.sql.catalog.iceberg_catalog", "org.apache.iceberg.spark.SparkCatalog") \ + .config("spark.sql.catalog.iceberg_catalog.type", catalog_config.get('type', 'hadoop')) \ + .config("spark.sql.catalog.iceberg_catalog.warehouse", catalog_config['warehouse']) \ + .getOrCreate() + + self.catalog_name = "iceberg_catalog" + + def create_table( + self, + database: str, + table_name: str, + df, + partition_by: Optional[List[str]] = None, + sort_order: Optional[List[str]] = None + ): + """ + Create Iceberg table from DataFrame. + + Args: + database: Database name + table_name: Table name + df: Spark DataFrame + partition_by: Partition columns + sort_order: Sort order for data files + """ + full_table_name = f"{self.catalog_name}.{database}.{table_name}" + + # Write DataFrame as Iceberg table + writer = df.writeTo(full_table_name).using("iceberg") + + if partition_by: + writer = writer.partitionedBy(*partition_by) + + if sort_order: + for col in sort_order: + writer = writer.sortedBy(col) + + writer.create() + + logger.info(f"Created Iceberg table: {full_table_name}") + + def incremental_upsert( + self, + database: str, + table_name: str, + df, + merge_keys: List[str], + update_columns: Optional[List[str]] = None + ): + """ + Perform incremental upsert using MERGE INTO. + + Args: + database: Database name + table_name: Table name + df: Spark DataFrame with updates + merge_keys: Columns to match on + update_columns: Columns to update (all if None) + """ + full_table_name = f"{self.catalog_name}.{database}.{table_name}" + + # Register DataFrame as temp view + df.createOrReplaceTempView("updates") + + # Build merge condition + merge_condition = " AND ".join([ + f"target.{key} = updates.{key}" for key in merge_keys + ]) + + # Build update set clause + if update_columns: + update_set = ", ".join([ + f"{col} = updates.{col}" for col in update_columns + ]) + else: + update_set = ", ".join([ + f"{col} = updates.{col}" for col in df.columns + ]) + + # Build insert values + insert_cols = ", ".join(df.columns) + insert_vals = ", ".join([f"updates.{col}" for col in df.columns]) + + # Execute merge + merge_query = f""" + MERGE INTO {full_table_name} AS target + USING updates + ON {merge_condition} + WHEN MATCHED THEN + UPDATE SET {update_set} + WHEN NOT MATCHED THEN + INSERT ({insert_cols}) + VALUES ({insert_vals}) + """ + + self.spark.sql(merge_query) + logger.info(f"Completed upsert for: {full_table_name}") + + def optimize_table( + self, + database: str, + table_name: str + ): + """ + Optimize Iceberg table by rewriting small files. + + Args: + database: Database name + table_name: Table name + """ + full_table_name = f"{self.catalog_name}.{database}.{table_name}" + + # Rewrite data files + self.spark.sql(f""" + CALL {self.catalog_name}.system.rewrite_data_files( + table => '{database}.{table_name}', + strategy => 'binpack', + options => map('target-file-size-bytes', '536870912') + ) + """) + + # Expire old snapshots (keep last 7 days) + self.spark.sql(f""" + CALL {self.catalog_name}.system.expire_snapshots( + table => '{database}.{table_name}', + older_than => DATE_SUB(CURRENT_DATE(), 7), + retain_last => 5 + ) + """) + + logger.info(f"Optimized table: {full_table_name}") + + def time_travel_query( + self, + database: str, + table_name: str, + snapshot_id: Optional[int] = None, + timestamp_ms: Optional[int] = None + ): + """ + Query historical snapshot of Iceberg table. + + Args: + database: Database name + table_name: Table name + snapshot_id: Specific snapshot ID + timestamp_ms: Timestamp in milliseconds + """ + full_table_name = f"{self.catalog_name}.{database}.{table_name}" + + if snapshot_id: + query = f"SELECT * FROM {full_table_name} VERSION AS OF {snapshot_id}" + elif timestamp_ms: + query = f"SELECT * FROM {full_table_name} TIMESTAMP AS OF {timestamp_ms}" + else: + query = f"SELECT * FROM {full_table_name}" + + return self.spark.sql(query) +``` + +### 7. Monitoring, Observability, and Cost Optimization + +**Pipeline Monitoring Framework** + +```python +# monitoring/pipeline_monitor.py +import logging +from dataclasses import dataclass +from datetime import datetime +from typing import Dict, List, Optional +import boto3 +import json + +logger = logging.getLogger(__name__) + +@dataclass +class PipelineMetrics: + """Data class for pipeline metrics.""" + pipeline_name: str + execution_id: str + start_time: datetime + end_time: Optional[datetime] + status: str # running, success, failed + records_processed: int + records_failed: int + data_size_bytes: int + execution_time_seconds: Optional[float] + error_message: Optional[str] = None + +class PipelineMonitor: + """Comprehensive pipeline monitoring and alerting.""" + + def __init__(self, config: Dict): + self.config = config + self.cloudwatch = boto3.client('cloudwatch') + self.sns = boto3.client('sns') + self.alert_topic_arn = config.get('sns_topic_arn') + + def track_pipeline_execution(self, metrics: PipelineMetrics): + """ + Track pipeline execution metrics in CloudWatch. + + Args: + metrics: Pipeline execution metrics + """ + namespace = f"DataPipeline/{metrics.pipeline_name}" + + metric_data = [ + { + 'MetricName': 'RecordsProcessed', + 'Value': metrics.records_processed, + 'Unit': 'Count', + 'Timestamp': metrics.start_time + }, + { + 'MetricName': 'RecordsFailed', + 'Value': metrics.records_failed, + 'Unit': 'Count', + 'Timestamp': metrics.start_time + }, + { + 'MetricName': 'DataSizeBytes', + 'Value': metrics.data_size_bytes, + 'Unit': 'Bytes', + 'Timestamp': metrics.start_time + } + ] + + if metrics.execution_time_seconds: + metric_data.append({ + 'MetricName': 'ExecutionTime', + 'Value': metrics.execution_time_seconds, + 'Unit': 'Seconds', + 'Timestamp': metrics.start_time + }) + + if metrics.status == 'success': + metric_data.append({ + 'MetricName': 'PipelineSuccess', + 'Value': 1, + 'Unit': 'Count', + 'Timestamp': metrics.start_time + }) + elif metrics.status == 'failed': + metric_data.append({ + 'MetricName': 'PipelineFailure', + 'Value': 1, + 'Unit': 'Count', + 'Timestamp': metrics.start_time + }) + + self.cloudwatch.put_metric_data( + Namespace=namespace, + MetricData=metric_data + ) + + logger.info(f"Tracked metrics for pipeline: {metrics.pipeline_name}") + + def send_alert( + self, + severity: str, + title: str, + message: str, + metadata: Optional[Dict] = None + ): + """ + Send alert notification via SNS. + + Args: + severity: "critical", "warning", or "info" + title: Alert title + message: Alert message + metadata: Additional context + """ + alert_payload = { + 'severity': severity, + 'title': title, + 'message': message, + 'timestamp': datetime.utcnow().isoformat(), + 'metadata': metadata or {} + } + + if self.alert_topic_arn: + self.sns.publish( + TopicArn=self.alert_topic_arn, + Subject=f"[{severity.upper()}] {title}", + Message=json.dumps(alert_payload, indent=2) + ) + logger.info(f"Sent {severity} alert: {title}") + + def check_data_freshness( + self, + table_path: str, + max_age_hours: int = 24 + ) -> bool: + """ + Check if data is fresh enough based on last update. + + Args: + table_path: Path to data table + max_age_hours: Maximum acceptable age in hours + """ + from deltalake import DeltaTable + from datetime import timedelta + + try: + dt = DeltaTable(table_path) + history = dt.history() + + if not history: + self.send_alert( + 'warning', + 'No Data History', + f'Table {table_path} has no history' + ) + return False + + last_update = history[0]['timestamp'] + age = datetime.utcnow() - last_update + + if age > timedelta(hours=max_age_hours): + self.send_alert( + 'warning', + 'Stale Data Detected', + f'Table {table_path} is {age.total_seconds() / 3600:.1f} hours old', + metadata={'table': table_path, 'last_update': last_update.isoformat()} + ) + return False + + return True + + except Exception as e: + logger.error(f"Freshness check failed: {e}") + return False + + def analyze_pipeline_performance( + self, + pipeline_name: str, + time_range_hours: int = 24 + ) -> Dict: + """ + Analyze pipeline performance over time period. + + Args: + pipeline_name: Name of pipeline to analyze + time_range_hours: Hours of history to analyze + """ + from datetime import timedelta + + end_time = datetime.utcnow() + start_time = end_time - timedelta(hours=time_range_hours) + + # Get metrics from CloudWatch + response = self.cloudwatch.get_metric_statistics( + Namespace=f"DataPipeline/{pipeline_name}", + MetricName='ExecutionTime', + StartTime=start_time, + EndTime=end_time, + Period=3600, # 1 hour + Statistics=['Average', 'Maximum', 'Minimum'] + ) + + datapoints = response.get('Datapoints', []) + + if not datapoints: + return {'status': 'no_data', 'message': 'No metrics available'} + + avg_execution_time = sum(dp['Average'] for dp in datapoints) / len(datapoints) + max_execution_time = max(dp['Maximum'] for dp in datapoints) + + performance_summary = { + 'pipeline_name': pipeline_name, + 'time_range_hours': time_range_hours, + 'avg_execution_time_seconds': avg_execution_time, + 'max_execution_time_seconds': max_execution_time, + 'datapoints': len(datapoints) + } + + # Alert if performance degraded + if avg_execution_time > 1800: # 30 minutes threshold + self.send_alert( + 'warning', + 'Pipeline Performance Degradation', + f'{pipeline_name} average execution time: {avg_execution_time:.1f}s', + metadata=performance_summary + ) + + return performance_summary + + +**Cost Optimization Strategies** + +```python +# cost_optimization/optimizer.py +import logging +from typing import Dict, List +from datetime import datetime, timedelta + +logger = logging.getLogger(__name__) + +class CostOptimizer: + """Pipeline cost optimization strategies.""" + + def __init__(self, config: Dict): + self.config = config + + def implement_partitioning_strategy( + self, + table_name: str, + partition_columns: List[str], + partition_type: str = "date" + ) -> Dict: + """ + Design optimal partitioning strategy to reduce query costs. + + Recommendations: + - Date partitioning: For time-series data, partition by date/timestamp + - User/Entity partitioning: For user-specific queries, partition by user_id + - Multi-level: Combine date + region for geographic data + - Avoid over-partitioning: Keep partitions > 1GB for best performance + """ + strategy = { + 'table_name': table_name, + 'partition_columns': partition_columns, + 'recommendations': [] + } + + if partition_type == "date": + strategy['recommendations'].extend([ + "Partition by day for daily queries, month for long-term analysis", + "Use partition pruning in queries: WHERE date = '2025-01-01'", + "Consider clustering by frequently filtered columns within partitions", + f"Estimated cost savings: 60-90% for date-range queries" + ]) + + logger.info(f"Partitioning strategy for {table_name}: {strategy}") + return strategy + + def optimize_file_sizes( + self, + table_path: str, + target_file_size_mb: int = 512 + ): + """ + Optimize file sizes to reduce metadata overhead and improve query performance. + + Best practices: + - Target file size: 512MB - 1GB for Parquet + - Avoid small files (<128MB) which increase metadata overhead + - Avoid very large files (>2GB) which reduce parallelism + """ + from deltalake import DeltaTable + + dt = DeltaTable(table_path) + + # Compact small files + dt.optimize.compact() + + logger.info(f"Optimized file sizes for {table_path}") + + return { + 'table_path': table_path, + 'target_file_size_mb': target_file_size_mb, + 'optimization': 'completed' + } + + def implement_lifecycle_policies( + self, + storage_path: str, + hot_tier_days: int = 30, + cold_tier_days: int = 90, + archive_days: int = 365 + ) -> Dict: + """ + Design storage lifecycle policies for cost optimization. + + Storage tiers (AWS S3 example): + - Standard: Frequent access (0-30 days) + - Infrequent Access: Occasional access (30-90 days) + - Glacier: Archive (90+ days) + + Cost savings: Up to 90% compared to Standard storage + """ + lifecycle_policy = { + 'storage_path': storage_path, + 'tiers': { + 'hot': { + 'days': hot_tier_days, + 'storage_class': 'STANDARD', + 'cost_per_gb': 0.023 + }, + 'warm': { + 'days': cold_tier_days - hot_tier_days, + 'storage_class': 'STANDARD_IA', + 'cost_per_gb': 0.0125 + }, + 'cold': { + 'days': archive_days - cold_tier_days, + 'storage_class': 'GLACIER', + 'cost_per_gb': 0.004 + } + }, + 'estimated_savings_percent': 70 + } + + logger.info(f"Lifecycle policy for {storage_path}: {lifecycle_policy}") + return lifecycle_policy + + def optimize_compute_resources( + self, + workload_type: str, + data_size_gb: float + ) -> Dict: + """ + Recommend optimal compute resources for workload. + + Args: + workload_type: "batch", "streaming", or "adhoc" + data_size_gb: Size of data to process + """ + if workload_type == "batch": + # Use scheduled spot instances for cost savings + recommendation = { + 'instance_type': 'c5.4xlarge', + 'instance_count': max(1, int(data_size_gb / 100)), + 'use_spot_instances': True, + 'estimated_cost_savings': '70%', + 'notes': 'Spot instances for non-time-critical batch jobs' + } + elif workload_type == "streaming": + # Use reserved or on-demand for reliability + recommendation = { + 'instance_type': 'r5.2xlarge', + 'instance_count': max(2, int(data_size_gb / 50)), + 'use_spot_instances': False, + 'estimated_cost_savings': '0%', + 'notes': 'On-demand for reliable streaming processing' + } + else: + # Adhoc queries - use serverless + recommendation = { + 'service': 'AWS Athena / BigQuery / Snowflake', + 'billing': 'pay-per-query', + 'estimated_cost': f'${data_size_gb * 0.005:.2f}', + 'notes': 'Serverless for unpredictable adhoc workloads' + } + + logger.info(f"Compute recommendation for {workload_type}: {recommendation}") + return recommendation +``` + +## Reference Examples + +### Example 1: Real-Time E-Commerce Analytics Pipeline + +**Purpose**: Process e-commerce events in real-time, enrich with user data, aggregate metrics, and serve to dashboards. + +**Architecture**: +- **Ingestion**: Kafka receives clickstream and transaction events +- **Processing**: Flink performs stateful stream processing with windowing +- **Storage**: Write to Iceberg for ad-hoc queries, Redis for real-time metrics +- **Orchestration**: Kubernetes manages Flink jobs +- **Monitoring**: Prometheus + Grafana for observability + +**Implementation**: + +```python +# Real-time e-commerce pipeline with Flink (PyFlink) +from pyflink.datastream import StreamExecutionEnvironment +from pyflink.datastream.connectors import FlinkKafkaConsumer, FlinkKafkaProducer +from pyflink.common.serialization import SimpleStringSchema +from pyflink.datastream.functions import MapFunction, KeyedProcessFunction +from pyflink.common.time import Time +from pyflink.common.typeinfo import Types +import json + +class EventEnrichment(MapFunction): + """Enrich events with additional context.""" + + def __init__(self, user_cache): + self.user_cache = user_cache + + def map(self, value): + event = json.loads(value) + user_id = event.get('user_id') + + # Enrich with user data from cache/database + if user_id and user_id in self.user_cache: + event['user_tier'] = self.user_cache[user_id]['tier'] + event['user_region'] = self.user_cache[user_id]['region'] + + return json.dumps(event) + +class RevenueAggregator(KeyedProcessFunction): + """Calculate rolling revenue metrics per user.""" + + def process_element(self, value, ctx): + event = json.loads(value) + + if event.get('event_type') == 'purchase': + revenue = event.get('amount', 0) + + # Emit aggregated metric + yield { + 'user_id': event['user_id'], + 'timestamp': ctx.timestamp(), + 'revenue': revenue, + 'window': 'last_hour' + } + +def create_ecommerce_pipeline(): + """Create real-time e-commerce analytics pipeline.""" + + env = StreamExecutionEnvironment.get_execution_environment() + env.set_parallelism(4) + + # Kafka consumer properties + kafka_props = { + 'bootstrap.servers': 'kafka:9092', + 'group.id': 'ecommerce-analytics' + } + + # Create Kafka source + kafka_consumer = FlinkKafkaConsumer( + topics='ecommerce-events', + deserialization_schema=SimpleStringSchema(), + properties=kafka_props + ) + + # Read stream + events = env.add_source(kafka_consumer) + + # Enrich events + user_cache = {} # In production, use Redis or other cache + enriched = events.map(EventEnrichment(user_cache)) + + # Calculate revenue per user (tumbling window) + revenue_metrics = ( + enriched + .key_by(lambda x: json.loads(x)['user_id']) + .window(Time.hours(1)) + .process(RevenueAggregator()) + ) + + # Write to Kafka for downstream consumption + kafka_producer = FlinkKafkaProducer( + topic='revenue-metrics', + serialization_schema=SimpleStringSchema(), + producer_config=kafka_props + ) + + revenue_metrics.map(lambda x: json.dumps(x)).add_sink(kafka_producer) + + # Execute + env.execute("E-Commerce Analytics Pipeline") + +if __name__ == "__main__": + create_ecommerce_pipeline() +``` + +### Example 2: Data Lakehouse with dbt Transformations + +**Purpose**: Build dimensional data warehouse on lakehouse architecture for analytics. + +**Complete Pipeline**: + +```python +# Complete lakehouse pipeline orchestration +from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.operators.bash import BashOperator +from datetime import datetime, timedelta + +def extract_and_load_to_lakehouse(): + """Extract from multiple sources and load to Delta Lake.""" + from storage.delta_lake_manager import DeltaLakeManager + from batch_ingestion import BatchDataIngester + + ingester = BatchDataIngester(config={}) + delta_manager = DeltaLakeManager(storage_path='s3://data-lakehouse/bronze') + + # Extract from PostgreSQL + orders_df = ingester.extract_from_database( + connection_string='postgresql://localhost:5432/ecommerce', + query='SELECT * FROM orders WHERE created_at >= CURRENT_DATE - INTERVAL \'1 day\'', + watermark_column='created_at', + last_watermark=datetime.now() - timedelta(days=1) + ) + + # Write to bronze layer (raw data) + delta_manager.create_or_update_table( + df=orders_df, + table_name='orders', + partition_columns=['order_date'], + mode='append' + ) + +with DAG( + 'lakehouse_analytics_pipeline', + schedule_interval='@daily', + start_date=datetime(2025, 1, 1), + catchup=False +) as dag: + + extract = PythonOperator( + task_id='extract_to_bronze', + python_callable=extract_and_load_to_lakehouse + ) + + # dbt transformation: bronze -> silver -> gold + dbt_silver = BashOperator( + task_id='dbt_silver_layer', + bash_command='dbt run --models silver.* --profiles-dir /opt/dbt' + ) + + dbt_gold = BashOperator( + task_id='dbt_gold_layer', + bash_command='dbt run --models gold.* --profiles-dir /opt/dbt' + ) + + dbt_test = BashOperator( + task_id='dbt_test', + bash_command='dbt test --profiles-dir /opt/dbt' + ) + + extract >> dbt_silver >> dbt_gold >> dbt_test +``` + +### Example 3: CDC Pipeline with Debezium and Kafka + +**Purpose**: Capture database changes in real-time and replicate to data warehouse. + +**Architecture**: MySQL -> Debezium -> Kafka -> Flink -> Snowflake + +```python +# CDC processing with Kafka consumer +from streaming_ingestion import StreamingDataIngester +import snowflake.connector + +def process_cdc_events(messages): + """Process CDC events from Debezium.""" + processed = [] + + for msg in messages: + event = msg['value'] + operation = event.get('op') # 'c'=create, 'u'=update, 'd'=delete + + if operation in ['c', 'u']: + # Insert or update + after = event.get('after', {}) + processed.append({ + 'key': after.get('id'), + 'value': { + 'operation': 'upsert', + 'table': event.get('source', {}).get('table'), + 'data': after, + 'timestamp': event.get('ts_ms') + } + }) + elif operation == 'd': + # Delete + before = event.get('before', {}) + processed.append({ + 'key': before.get('id'), + 'value': { + 'operation': 'delete', + 'table': event.get('source', {}).get('table'), + 'id': before.get('id'), + 'timestamp': event.get('ts_ms') + } + }) + + return processed + +def sync_to_snowflake(processed_events): + """Sync CDC events to Snowflake.""" + conn = snowflake.connector.connect( + user='user', + password='pass', + account='account', + warehouse='COMPUTE_WH', + database='analytics', + schema='replicated' + ) + + cursor = conn.cursor() + + for event in processed_events: + if event['value']['operation'] == 'upsert': + # Merge into Snowflake + data = event['value']['data'] + table = event['value']['table'] + + merge_sql = f""" + MERGE INTO {table} AS target + USING (SELECT {', '.join([f"'{v}' AS {k}" for k, v in data.items()])}) AS source + ON target.id = source.id + WHEN MATCHED THEN UPDATE SET {', '.join([f"{k} = source.{k}" for k in data.keys()])} + WHEN NOT MATCHED THEN INSERT ({', '.join(data.keys())}) + VALUES ({', '.join([f"source.{k}" for k in data.keys()])}) + """ + cursor.execute(merge_sql) + + elif event['value']['operation'] == 'delete': + table = event['value']['table'] + id_val = event['value']['id'] + cursor.execute(f"DELETE FROM {table} WHERE id = {id_val}") + + conn.commit() + cursor.close() + conn.close() + +# Run CDC pipeline +kafka_config = { + 'bootstrap_servers': 'kafka:9092', + 'consumer_group': 'cdc-replication', + 'transactional_id': 'cdc-txn' +} + +ingester = StreamingDataIngester(kafka_config) +ingester.consume_and_process( + topics=['mysql.ecommerce.orders', 'mysql.ecommerce.customers'], + process_func=process_cdc_events, + batch_size=100 +) +``` + +## Output Format + +Deliver a comprehensive data pipeline solution with the following components: + +### 1. Architecture Documentation +- **Architecture diagram** showing data flow from sources to destinations +- **Technology stack** with justification for each component +- **Scalability analysis** with expected throughput and growth patterns +- **Failure modes** and recovery strategies + +### 2. Implementation Code +- **Ingestion layer**: Batch and streaming data ingestion code +- **Transformation layer**: dbt models or Spark jobs for data transformations +- **Orchestration**: Airflow/Prefect DAGs with dependency management +- **Storage**: Delta Lake/Iceberg table management code +- **Data quality**: Great Expectations suites and validation logic + +### 3. Configuration Files +- **Orchestration configs**: DAG definitions, schedules, retry policies +- **dbt project**: models, sources, tests, documentation +- **Infrastructure**: Docker Compose, Kubernetes manifests, Terraform for cloud resources +- **Environment configs**: Development, staging, production configurations + +### 4. Monitoring and Observability +- **Metrics collection**: Pipeline execution metrics, data quality scores +- **Alerting rules**: Thresholds for failures, performance degradation, data freshness +- **Dashboards**: Grafana/CloudWatch dashboards for pipeline monitoring +- **Logging strategy**: Structured logging with correlation IDs + +### 5. Operations Guide +- **Deployment procedures**: How to deploy pipeline updates +- **Troubleshooting guide**: Common issues and resolution steps +- **Scaling guide**: How to scale for increased data volume +- **Cost optimization**: Strategies implemented and potential savings +- **Disaster recovery**: Backup and recovery procedures + +### Success Criteria +- [ ] Pipeline processes data within defined SLA (latency requirements met) +- [ ] Data quality checks pass with >99% success rate +- [ ] Pipeline handles failures gracefully with automatic retry and alerting +- [ ] Comprehensive monitoring shows pipeline health and performance +- [ ] Documentation enables other engineers to understand and maintain pipeline +- [ ] Cost optimization strategies reduce infrastructure costs by 30-50% +- [ ] Schema evolution handled without pipeline downtime +- [ ] End-to-end data lineage tracked from source to destination diff --git a/tools/data-validation.md b/tools/data-validation.md index 750f535..6d372af 100644 --- a/tools/data-validation.md +++ b/tools/data-validation.md @@ -1,60 +1,1674 @@ ---- -model: sonnet ---- - # Data Validation Pipeline +You are a data validation and quality assurance expert specializing in comprehensive data validation frameworks, quality monitoring systems, and anomaly detection. You excel at implementing robust validation pipelines using modern tools like Pydantic v2, Great Expectations, and custom validation frameworks to ensure data integrity, consistency, and reliability across diverse data systems and formats. + +## Context + +The user needs a comprehensive data validation system that ensures data quality throughout the entire data lifecycle. Focus on building scalable validation pipelines that catch issues early, provide clear error reporting, support both batch and real-time validation, and integrate seamlessly with existing data infrastructure while maintaining high performance and extensibility. + +## Requirements + Create a comprehensive data validation system for: $ARGUMENTS -Implement validation including: +## Instructions -1. **Schema Validation**: - - Pydantic models for structure - - JSON Schema generation - - Type checking and coercion - - Nested object validation - - Custom validators +### 1. Schema Validation and Data Modeling -2. **Data Quality Checks**: - - Null/missing value handling - - Outlier detection - - Statistical validation - - Business rule enforcement - - Referential integrity +Design and implement schema validation using modern frameworks that enforce data structure, types, and business rules at the point of data entry. -3. **Data Profiling**: - - Automatic type inference - - Distribution analysis - - Cardinality checks - - Pattern detection - - Anomaly identification +**Pydantic v2 Model Implementation** +```python +from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic.functional_validators import AfterValidator +from typing import Optional, List, Dict, Any +from datetime import datetime, date +from decimal import Decimal +import re +from enum import Enum -4. **Validation Rules**: - - Field-level constraints - - Cross-field validation - - Temporal consistency - - Format validation (email, phone, etc.) - - Custom business logic +class CustomerStatus(str, Enum): + ACTIVE = "active" + INACTIVE = "inactive" + SUSPENDED = "suspended" + PENDING = "pending" -5. **Error Handling**: - - Detailed error messages - - Error categorization - - Partial validation support - - Error recovery strategies - - Validation reports +class Address(BaseModel): + street: str = Field(..., min_length=1, max_length=200) + city: str = Field(..., min_length=1, max_length=100) + state: str = Field(..., pattern=r'^[A-Z]{2}$') + zip_code: str = Field(..., pattern=r'^\d{5}(-\d{4})?$') + country: str = Field(default="US", pattern=r'^[A-Z]{2}$') -6. **Performance**: - - Streaming validation - - Batch processing - - Parallel validation - - Caching strategies - - Incremental validation + @field_validator('state') + def validate_state(cls, v, info): + valid_states = ['CA', 'NY', 'TX', 'FL', 'IL', 'PA'] # Add all valid states + if v not in valid_states: + raise ValueError(f'Invalid state code: {v}') + return v -7. **Integration**: - - API endpoint validation - - Database constraints - - Message queue validation - - File upload validation - - Real-time validation +class Customer(BaseModel): + customer_id: str = Field(..., pattern=r'^CUST-\d{8}$') + email: str = Field(..., pattern=r'^[\w\.-]+@[\w\.-]+\.\w+$') + phone: Optional[str] = Field(None, pattern=r'^\+?1?\d{10,14}$') + first_name: str = Field(..., min_length=1, max_length=50) + last_name: str = Field(..., min_length=1, max_length=50) + date_of_birth: date + registration_date: datetime + status: CustomerStatus + credit_limit: Decimal = Field(..., ge=0, le=1000000) + addresses: List[Address] = Field(..., min_items=1, max_items=5) + metadata: Dict[str, Any] = Field(default_factory=dict) -Include data quality metrics, monitoring dashboards, and alerting. Make it extensible for custom validation rules. + @field_validator('email') + def validate_email_domain(cls, v): + blocked_domains = ['tempmail.com', 'throwaway.email'] + domain = v.split('@')[-1] + if domain in blocked_domains: + raise ValueError(f'Email domain {domain} is not allowed') + return v.lower() + + @field_validator('date_of_birth') + def validate_age(cls, v): + today = date.today() + age = today.year - v.year - ((today.month, today.day) < (v.month, v.day)) + if age < 18: + raise ValueError('Customer must be at least 18 years old') + if age > 120: + raise ValueError('Invalid date of birth') + return v + + @model_validator(mode='after') + def validate_registration_after_birth(self): + if self.registration_date.date() < self.date_of_birth: + raise ValueError('Registration date cannot be before birth date') + return self + + class Config: + json_schema_extra = { + "example": { + "customer_id": "CUST-12345678", + "email": "john.doe@example.com", + "first_name": "John", + "last_name": "Doe", + "date_of_birth": "1990-01-15", + "registration_date": "2024-01-01T10:00:00Z", + "status": "active", + "credit_limit": 5000.00, + "addresses": [ + { + "street": "123 Main St", + "city": "San Francisco", + "state": "CA", + "zip_code": "94105" + } + ] + } + } +``` + +**JSON Schema Generation and Validation** +```python +import json +from jsonschema import validate, ValidationError, Draft7Validator + +# Generate JSON Schema from Pydantic model +customer_schema = Customer.model_json_schema() + +# Save schema for external validation +with open('customer_schema.json', 'w') as f: + json.dump(customer_schema, f, indent=2) + +# Validate raw JSON data +def validate_json_data(data: dict, schema: dict) -> tuple[bool, list]: + """Validate JSON data against schema and return errors.""" + validator = Draft7Validator(schema) + errors = list(validator.iter_errors(data)) + + if errors: + error_messages = [] + for error in errors: + path = ' -> '.join(str(p) for p in error.path) + error_messages.append(f"{path}: {error.message}") + return False, error_messages + return True, [] + +# Custom validator with business rules +def validate_customer_data(data: dict) -> Customer: + """Validate and parse customer data with comprehensive error handling.""" + try: + customer = Customer.model_validate(data) + + # Additional business rule validations + if customer.status == CustomerStatus.SUSPENDED and customer.credit_limit > 0: + raise ValueError("Suspended customers cannot have credit limit > 0") + + return customer + except ValidationError as e: + # Format errors for better readability + errors = [] + for error in e.errors(): + location = ' -> '.join(str(loc) for loc in error['loc']) + errors.append(f"{location}: {error['msg']}") + raise ValueError(f"Validation failed:\n" + '\n'.join(errors)) +``` + +### 2. Data Quality Dimensions and Monitoring + +Implement comprehensive data quality checks across all critical dimensions to ensure data fitness for use. + +**Data Quality Framework Implementation** +```python +import pandas as pd +import numpy as np +from typing import Dict, List, Tuple, Any +from dataclasses import dataclass +from datetime import datetime, timedelta +import hashlib + +@dataclass +class DataQualityMetrics: + completeness: float + accuracy: float + consistency: float + timeliness: float + uniqueness: float + validity: float + + @property + def overall_score(self) -> float: + """Calculate weighted overall data quality score.""" + weights = { + 'completeness': 0.25, + 'accuracy': 0.20, + 'consistency': 0.20, + 'timeliness': 0.15, + 'uniqueness': 0.10, + 'validity': 0.10 + } + return sum(getattr(self, dim) * weight + for dim, weight in weights.items()) + +class DataQualityValidator: + """Comprehensive data quality validation framework.""" + + def __init__(self, df: pd.DataFrame, schema: Dict[str, Any]): + self.df = df + self.schema = schema + self.validation_results = {} + + def check_completeness(self) -> float: + """Check for missing values and required fields.""" + total_cells = self.df.size + missing_cells = self.df.isna().sum().sum() + + # Check required fields + required_fields = [col for col, spec in self.schema.items() + if spec.get('required', False)] + required_complete = all(col in self.df.columns for col in required_fields) + + completeness_score = (total_cells - missing_cells) / total_cells if total_cells > 0 else 0 + + # Adjust score if required fields are missing + if not required_complete: + completeness_score *= 0.5 + + self.validation_results['completeness'] = { + 'score': completeness_score, + 'missing_cells': int(missing_cells), + 'total_cells': int(total_cells), + 'missing_by_column': self.df.isna().sum().to_dict() + } + + return completeness_score + + def check_accuracy(self, reference_data: pd.DataFrame = None) -> float: + """Check data accuracy against reference data or business rules.""" + accuracy_checks = [] + + # Format validations + for col, spec in self.schema.items(): + if col not in self.df.columns: + continue + + if 'pattern' in spec: + pattern = spec['pattern'] + valid_format = self.df[col].astype(str).str.match(pattern) + accuracy_checks.append(valid_format.mean()) + + if 'range' in spec: + min_val, max_val = spec['range'] + in_range = self.df[col].between(min_val, max_val) + accuracy_checks.append(in_range.mean()) + + # Reference data comparison if available + if reference_data is not None: + common_cols = set(self.df.columns) & set(reference_data.columns) + for col in common_cols: + matches = (self.df[col] == reference_data[col]).mean() + accuracy_checks.append(matches) + + accuracy_score = np.mean(accuracy_checks) if accuracy_checks else 1.0 + + self.validation_results['accuracy'] = { + 'score': accuracy_score, + 'checks_performed': len(accuracy_checks) + } + + return accuracy_score + + def check_consistency(self) -> float: + """Check internal consistency and cross-field validation.""" + consistency_checks = [] + + # Check for duplicate records + duplicate_ratio = self.df.duplicated().sum() / len(self.df) + consistency_checks.append(1 - duplicate_ratio) + + # Cross-field consistency rules + if 'start_date' in self.df.columns and 'end_date' in self.df.columns: + date_consistency = (self.df['start_date'] <= self.df['end_date']).mean() + consistency_checks.append(date_consistency) + + # Check referential integrity + if 'foreign_keys' in self.schema: + for fk_config in self.schema['foreign_keys']: + column = fk_config['column'] + reference_values = fk_config['reference_values'] + if column in self.df.columns: + integrity_check = self.df[column].isin(reference_values).mean() + consistency_checks.append(integrity_check) + + consistency_score = np.mean(consistency_checks) if consistency_checks else 1.0 + + self.validation_results['consistency'] = { + 'score': consistency_score, + 'duplicate_count': int(self.df.duplicated().sum()), + 'checks_performed': len(consistency_checks) + } + + return consistency_score + + def check_timeliness(self, max_age_days: int = 30) -> float: + """Check data freshness and timeliness.""" + timestamp_cols = self.df.select_dtypes(include=['datetime64']).columns + + if len(timestamp_cols) == 0: + return 1.0 + + timeliness_scores = [] + current_time = pd.Timestamp.now() + + for col in timestamp_cols: + # Calculate age of records + age_days = (current_time - self.df[col]).dt.days + within_threshold = (age_days <= max_age_days).mean() + timeliness_scores.append(within_threshold) + + timeliness_score = np.mean(timeliness_scores) + + self.validation_results['timeliness'] = { + 'score': timeliness_score, + 'max_age_days': max_age_days, + 'timestamp_columns': list(timestamp_cols) + } + + return timeliness_score + + def check_uniqueness(self, unique_columns: List[str] = None) -> float: + """Check uniqueness constraints.""" + if unique_columns is None: + unique_columns = [col for col, spec in self.schema.items() + if spec.get('unique', False)] + + if not unique_columns: + return 1.0 + + uniqueness_scores = [] + + for col in unique_columns: + if col in self.df.columns: + unique_ratio = self.df[col].nunique() / len(self.df) + uniqueness_scores.append(unique_ratio) + + uniqueness_score = np.mean(uniqueness_scores) if uniqueness_scores else 1.0 + + self.validation_results['uniqueness'] = { + 'score': uniqueness_score, + 'checked_columns': unique_columns + } + + return uniqueness_score + + def check_validity(self) -> float: + """Check data validity against defined schemas and types.""" + validity_checks = [] + + for col, spec in self.schema.items(): + if col not in self.df.columns: + continue + + # Type validation + expected_type = spec.get('type') + if expected_type: + if expected_type == 'numeric': + valid_type = pd.to_numeric(self.df[col], errors='coerce').notna() + elif expected_type == 'datetime': + valid_type = pd.to_datetime(self.df[col], errors='coerce').notna() + elif expected_type == 'string': + valid_type = self.df[col].apply(lambda x: isinstance(x, str)) + else: + valid_type = pd.Series([True] * len(self.df)) + + validity_checks.append(valid_type.mean()) + + # Enum validation + if 'enum' in spec: + valid_values = self.df[col].isin(spec['enum']) + validity_checks.append(valid_values.mean()) + + validity_score = np.mean(validity_checks) if validity_checks else 1.0 + + self.validation_results['validity'] = { + 'score': validity_score, + 'checks_performed': len(validity_checks) + } + + return validity_score + + def run_full_validation(self) -> DataQualityMetrics: + """Run all data quality checks and return comprehensive metrics.""" + metrics = DataQualityMetrics( + completeness=self.check_completeness(), + accuracy=self.check_accuracy(), + consistency=self.check_consistency(), + timeliness=self.check_timeliness(), + uniqueness=self.check_uniqueness(), + validity=self.check_validity() + ) + + self.validation_results['overall'] = { + 'score': metrics.overall_score, + 'timestamp': datetime.now().isoformat() + } + + return metrics +``` + +### 3. Great Expectations Implementation + +Set up production-grade data validation using Great Expectations for comprehensive testing and documentation. + +**Great Expectations Configuration** +```python +import great_expectations as gx +from great_expectations.checkpoint import Checkpoint +from great_expectations.core.batch import BatchRequest +from great_expectations.core.yaml_handler import YAMLHandler +import yaml + +class GreatExpectationsValidator: + """Production-grade data validation with Great Expectations.""" + + def __init__(self, project_root: str = "./great_expectations"): + self.context = gx.get_context(project_root=project_root) + + def create_datasource(self, name: str, connection_string: str = None): + """Create a datasource for validation.""" + if connection_string: + # SQL datasource + datasource_config = { + "name": name, + "class_name": "Datasource", + "execution_engine": { + "class_name": "SqlAlchemyExecutionEngine", + "connection_string": connection_string, + }, + "data_connectors": { + "default_inferred_data_connector_name": { + "class_name": "InferredAssetSqlDataConnector", + "include_schema_name": True, + } + } + } + else: + # Pandas datasource + datasource_config = { + "name": name, + "class_name": "Datasource", + "execution_engine": { + "class_name": "PandasExecutionEngine", + }, + "data_connectors": { + "default_runtime_data_connector_name": { + "class_name": "RuntimeDataConnector", + "batch_identifiers": ["default_identifier_name"], + } + } + } + + self.context.add_datasource(**datasource_config) + return self.context.get_datasource(name) + + def create_expectation_suite(self, suite_name: str): + """Create an expectation suite for validation rules.""" + suite = self.context.create_expectation_suite( + expectation_suite_name=suite_name, + overwrite_existing=True + ) + return suite + + def build_customer_expectations(self, batch_request): + """Build comprehensive expectations for customer data.""" + validator = self.context.get_validator( + batch_request=batch_request, + expectation_suite_name="customer_validation_suite" + ) + + # Table-level expectations + validator.expect_table_row_count_to_be_between(min_value=1, max_value=1000000) + validator.expect_table_column_count_to_equal(value=12) + + # Column existence + required_columns = [ + "customer_id", "email", "first_name", "last_name", + "registration_date", "status", "credit_limit" + ] + for column in required_columns: + validator.expect_column_to_exist(column=column) + + # Customer ID validations + validator.expect_column_values_to_not_be_null(column="customer_id") + validator.expect_column_values_to_be_unique(column="customer_id") + validator.expect_column_values_to_match_regex( + column="customer_id", + regex=r"^CUST-\d{8}$" + ) + + # Email validations + validator.expect_column_values_to_not_be_null(column="email") + validator.expect_column_values_to_be_unique(column="email") + validator.expect_column_values_to_match_regex( + column="email", + regex=r"^[\w\.-]+@[\w\.-]+\.\w+$" + ) + + # Name validations + validator.expect_column_value_lengths_to_be_between( + column="first_name", + min_value=1, + max_value=50 + ) + validator.expect_column_value_lengths_to_be_between( + column="last_name", + min_value=1, + max_value=50 + ) + + # Status validation + validator.expect_column_values_to_be_in_set( + column="status", + value_set=["active", "inactive", "suspended", "pending"] + ) + + # Credit limit validation + validator.expect_column_values_to_be_between( + column="credit_limit", + min_value=0, + max_value=1000000 + ) + validator.expect_column_mean_to_be_between( + column="credit_limit", + min_value=1000, + max_value=50000 + ) + + # Date validations + validator.expect_column_values_to_be_dateutil_parseable( + column="registration_date" + ) + validator.expect_column_values_to_be_increasing( + column="registration_date", + strictly=False + ) + + # Statistical expectations + validator.expect_column_stdev_to_be_between( + column="credit_limit", + min_value=100, + max_value=10000 + ) + + # Save expectations + validator.save_expectation_suite(discard_failed_expectations=False) + + return validator + + def create_checkpoint(self, checkpoint_name: str, suite_name: str): + """Create a checkpoint for automated validation.""" + checkpoint_config = { + "name": checkpoint_name, + "config_version": 1.0, + "class_name": "Checkpoint", + "expectation_suite_name": suite_name, + "action_list": [ + { + "name": "store_validation_result", + "action": { + "class_name": "StoreValidationResultAction" + } + }, + { + "name": "store_evaluation_params", + "action": { + "class_name": "StoreEvaluationParametersAction" + } + }, + { + "name": "update_data_docs", + "action": { + "class_name": "UpdateDataDocsAction" + } + } + ] + } + + self.context.add_checkpoint(**checkpoint_config) + return self.context.get_checkpoint(checkpoint_name) + + def run_validation(self, checkpoint_name: str, batch_request): + """Run validation checkpoint and return results.""" + checkpoint = self.context.get_checkpoint(checkpoint_name) + checkpoint_result = checkpoint.run( + batch_request=batch_request, + run_name=f"validation_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + ) + + return { + 'success': checkpoint_result.success, + 'statistics': checkpoint_result.run_results, + 'failed_expectations': self._extract_failed_expectations(checkpoint_result) + } + + def _extract_failed_expectations(self, checkpoint_result): + """Extract failed expectations from checkpoint results.""" + failed = [] + for result in checkpoint_result.run_results.values(): + for expectation_result in result['validation_result'].results: + if not expectation_result.success: + failed.append({ + 'expectation': expectation_result.expectation_config.expectation_type, + 'kwargs': expectation_result.expectation_config.kwargs, + 'result': expectation_result.result + }) + return failed +``` + +### 4. Real-time and Streaming Validation + +Implement validation for real-time data streams and event-driven architectures. + +**Streaming Validation Framework** +```python +import asyncio +from typing import AsyncIterator, Callable, Optional +from dataclasses import dataclass, field +import json +from kafka import KafkaConsumer, KafkaProducer +from kafka.errors import KafkaError +import aioredis +from datetime import datetime + +@dataclass +class ValidationResult: + record_id: str + timestamp: datetime + is_valid: bool + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + metadata: Dict[str, Any] = field(default_factory=dict) + +class StreamingValidator: + """Real-time streaming data validation framework.""" + + def __init__( + self, + kafka_bootstrap_servers: str, + redis_url: str = "redis://localhost:6379", + dead_letter_topic: str = "validation_errors" + ): + self.kafka_servers = kafka_bootstrap_servers + self.redis_url = redis_url + self.dead_letter_topic = dead_letter_topic + self.validators: Dict[str, Callable] = {} + self.metrics_cache = None + + async def initialize(self): + """Initialize connections to streaming infrastructure.""" + self.redis = await aioredis.create_redis_pool(self.redis_url) + + self.producer = KafkaProducer( + bootstrap_servers=self.kafka_servers, + value_serializer=lambda v: json.dumps(v).encode('utf-8'), + key_serializer=lambda k: k.encode('utf-8') if k else None + ) + + def register_validator(self, record_type: str, validator: Callable): + """Register a validator for a specific record type.""" + self.validators[record_type] = validator + + async def validate_stream( + self, + topic: str, + consumer_group: str, + batch_size: int = 100 + ) -> AsyncIterator[List[ValidationResult]]: + """Validate streaming data from Kafka topic.""" + consumer = KafkaConsumer( + topic, + bootstrap_servers=self.kafka_servers, + group_id=consumer_group, + value_deserializer=lambda m: json.loads(m.decode('utf-8')), + enable_auto_commit=False, + max_poll_records=batch_size + ) + + try: + while True: + messages = consumer.poll(timeout_ms=1000) + + if messages: + batch_results = [] + + for tp, records in messages.items(): + for record in records: + result = await self._validate_record(record.value) + batch_results.append(result) + + # Handle invalid records + if not result.is_valid: + await self._send_to_dead_letter(record.value, result) + + # Update metrics + await self._update_metrics(result) + + # Commit offsets after successful processing + consumer.commit() + + yield batch_results + + except KafkaError as e: + print(f"Kafka error: {e}") + raise + finally: + consumer.close() + + async def _validate_record(self, record: Dict) -> ValidationResult: + """Validate a single record.""" + record_type = record.get('type', 'unknown') + record_id = record.get('id', str(datetime.now().timestamp())) + + result = ValidationResult( + record_id=record_id, + timestamp=datetime.now(), + is_valid=True + ) + + # Apply type-specific validator + if record_type in self.validators: + try: + validator = self.validators[record_type] + validation_output = await validator(record) + + if isinstance(validation_output, dict): + result.is_valid = validation_output.get('is_valid', True) + result.errors = validation_output.get('errors', []) + result.warnings = validation_output.get('warnings', []) + + except Exception as e: + result.is_valid = False + result.errors.append(f"Validation error: {str(e)}") + else: + result.warnings.append(f"No validator registered for type: {record_type}") + + return result + + async def _send_to_dead_letter(self, record: Dict, result: ValidationResult): + """Send invalid records to dead letter queue.""" + dead_letter_record = { + 'original_record': record, + 'validation_result': { + 'record_id': result.record_id, + 'timestamp': result.timestamp.isoformat(), + 'errors': result.errors, + 'warnings': result.warnings + }, + 'processing_timestamp': datetime.now().isoformat() + } + + future = self.producer.send( + self.dead_letter_topic, + key=result.record_id, + value=dead_letter_record + ) + + try: + await asyncio.get_event_loop().run_in_executor( + None, future.get, 10 # 10 second timeout + ) + except KafkaError as e: + print(f"Failed to send to dead letter queue: {e}") + + async def _update_metrics(self, result: ValidationResult): + """Update validation metrics in Redis.""" + pipeline = self.redis.pipeline() + + # Increment counters + if result.is_valid: + pipeline.incr('validation:valid_count') + else: + pipeline.incr('validation:invalid_count') + + # Track error types + for error in result.errors: + error_type = error.split(':')[0] if ':' in error else 'unknown' + pipeline.hincrby('validation:error_types', error_type, 1) + + # Update recent validations list + pipeline.lpush( + 'validation:recent', + json.dumps({ + 'record_id': result.record_id, + 'timestamp': result.timestamp.isoformat(), + 'is_valid': result.is_valid + }) + ) + pipeline.ltrim('validation:recent', 0, 999) # Keep last 1000 + + await pipeline.execute() + + async def get_metrics(self) -> Dict[str, Any]: + """Retrieve current validation metrics.""" + valid_count = await self.redis.get('validation:valid_count') or 0 + invalid_count = await self.redis.get('validation:invalid_count') or 0 + error_types = await self.redis.hgetall('validation:error_types') + + total = int(valid_count) + int(invalid_count) + + return { + 'total_processed': total, + 'valid_count': int(valid_count), + 'invalid_count': int(invalid_count), + 'success_rate': int(valid_count) / total if total > 0 else 0, + 'error_distribution': { + k.decode(): int(v) for k, v in error_types.items() + }, + 'timestamp': datetime.now().isoformat() + } + +# Example custom validator for streaming data +async def validate_transaction(record: Dict) -> Dict: + """Custom validator for transaction records.""" + errors = [] + warnings = [] + + # Required field validation + required_fields = ['transaction_id', 'amount', 'timestamp', 'customer_id'] + for field in required_fields: + if field not in record: + errors.append(f"Missing required field: {field}") + + # Amount validation + if 'amount' in record: + amount = record['amount'] + if not isinstance(amount, (int, float)): + errors.append("Amount must be numeric") + elif amount <= 0: + errors.append("Amount must be positive") + elif amount > 100000: + warnings.append("Unusually high transaction amount") + + # Timestamp validation + if 'timestamp' in record: + try: + ts = datetime.fromisoformat(record['timestamp']) + if ts > datetime.now(): + errors.append("Transaction timestamp is in the future") + except: + errors.append("Invalid timestamp format") + + return { + 'is_valid': len(errors) == 0, + 'errors': errors, + 'warnings': warnings + } +``` + +### 5. Anomaly Detection and Data Profiling + +Implement statistical anomaly detection and automated data profiling for quality monitoring. + +**Anomaly Detection System** +```python +import numpy as np +from scipy import stats +from sklearn.ensemble import IsolationForest +from sklearn.preprocessing import StandardScaler +import pandas as pd + +class AnomalyDetector: + """Multi-method anomaly detection for data quality monitoring.""" + + def __init__(self, contamination: float = 0.1): + self.contamination = contamination + self.models = {} + self.scalers = {} + self.thresholds = {} + + def detect_statistical_anomalies( + self, + df: pd.DataFrame, + columns: List[str], + method: str = 'zscore' + ) -> pd.DataFrame: + """Detect anomalies using statistical methods.""" + anomalies = pd.DataFrame(index=df.index) + + for col in columns: + if col not in df.columns: + continue + + if method == 'zscore': + z_scores = np.abs(stats.zscore(df[col].dropna())) + anomalies[f'{col}_anomaly'] = z_scores > 3 + + elif method == 'iqr': + Q1 = df[col].quantile(0.25) + Q3 = df[col].quantile(0.75) + IQR = Q3 - Q1 + lower = Q1 - 1.5 * IQR + upper = Q3 + 1.5 * IQR + anomalies[f'{col}_anomaly'] = ~df[col].between(lower, upper) + + elif method == 'mad': # Median Absolute Deviation + median = df[col].median() + mad = np.median(np.abs(df[col] - median)) + modified_z = 0.6745 * (df[col] - median) / mad + anomalies[f'{col}_anomaly'] = np.abs(modified_z) > 3.5 + + anomalies['is_anomaly'] = anomalies.any(axis=1) + return anomalies + + def train_isolation_forest( + self, + df: pd.DataFrame, + feature_columns: List[str] + ): + """Train Isolation Forest for multivariate anomaly detection.""" + # Prepare data + X = df[feature_columns].fillna(df[feature_columns].mean()) + + # Scale features + scaler = StandardScaler() + X_scaled = scaler.fit_transform(X) + + # Train model + model = IsolationForest( + contamination=self.contamination, + random_state=42, + n_estimators=100 + ) + model.fit(X_scaled) + + # Store model and scaler + model_key = '_'.join(feature_columns) + self.models[model_key] = model + self.scalers[model_key] = scaler + + return model + + def detect_multivariate_anomalies( + self, + df: pd.DataFrame, + feature_columns: List[str] + ) -> np.ndarray: + """Detect anomalies using trained Isolation Forest.""" + model_key = '_'.join(feature_columns) + + if model_key not in self.models: + raise ValueError(f"No model trained for features: {feature_columns}") + + model = self.models[model_key] + scaler = self.scalers[model_key] + + X = df[feature_columns].fillna(df[feature_columns].mean()) + X_scaled = scaler.transform(X) + + # Predict anomalies (-1 for anomalies, 1 for normal) + predictions = model.predict(X_scaled) + anomaly_scores = model.score_samples(X_scaled) + + return predictions == -1, anomaly_scores + + def detect_temporal_anomalies( + self, + df: pd.DataFrame, + date_column: str, + value_column: str, + window_size: int = 7 + ) -> pd.DataFrame: + """Detect anomalies in time series data.""" + df = df.sort_values(date_column) + + # Calculate rolling statistics + rolling_mean = df[value_column].rolling(window=window_size).mean() + rolling_std = df[value_column].rolling(window=window_size).std() + + # Define bounds + upper_bound = rolling_mean + (2 * rolling_std) + lower_bound = rolling_mean - (2 * rolling_std) + + # Detect anomalies + anomalies = pd.DataFrame({ + 'value': df[value_column], + 'rolling_mean': rolling_mean, + 'upper_bound': upper_bound, + 'lower_bound': lower_bound, + 'is_anomaly': ~df[value_column].between(lower_bound, upper_bound) + }) + + return anomalies + +class DataProfiler: + """Automated data profiling for quality assessment.""" + + def profile_dataset(self, df: pd.DataFrame) -> Dict[str, Any]: + """Generate comprehensive data profile.""" + profile = { + 'basic_info': self._get_basic_info(df), + 'column_profiles': self._profile_columns(df), + 'correlations': self._calculate_correlations(df), + 'patterns': self._detect_patterns(df), + 'quality_issues': self._identify_quality_issues(df) + } + + return profile + + def _get_basic_info(self, df: pd.DataFrame) -> Dict: + """Get basic dataset information.""" + return { + 'row_count': len(df), + 'column_count': len(df.columns), + 'memory_usage': df.memory_usage(deep=True).sum() / 1024**2, # MB + 'duplicate_rows': df.duplicated().sum(), + 'missing_cells': df.isna().sum().sum(), + 'missing_percentage': (df.isna().sum().sum() / df.size) * 100 + } + + def _profile_columns(self, df: pd.DataFrame) -> Dict: + """Profile individual columns.""" + profiles = {} + + for col in df.columns: + col_profile = { + 'dtype': str(df[col].dtype), + 'missing_count': df[col].isna().sum(), + 'missing_percentage': (df[col].isna().sum() / len(df)) * 100, + 'unique_count': df[col].nunique(), + 'unique_percentage': (df[col].nunique() / len(df)) * 100 + } + + # Numeric column statistics + if pd.api.types.is_numeric_dtype(df[col]): + col_profile.update({ + 'mean': df[col].mean(), + 'median': df[col].median(), + 'std': df[col].std(), + 'min': df[col].min(), + 'max': df[col].max(), + 'q1': df[col].quantile(0.25), + 'q3': df[col].quantile(0.75), + 'skewness': df[col].skew(), + 'kurtosis': df[col].kurtosis(), + 'zeros': (df[col] == 0).sum(), + 'negative': (df[col] < 0).sum() + }) + + # String column statistics + elif pd.api.types.is_string_dtype(df[col]): + col_profile.update({ + 'min_length': df[col].str.len().min(), + 'max_length': df[col].str.len().max(), + 'avg_length': df[col].str.len().mean(), + 'empty_strings': (df[col] == '').sum(), + 'most_common': df[col].value_counts().head(5).to_dict() + }) + + # Datetime column statistics + elif pd.api.types.is_datetime64_any_dtype(df[col]): + col_profile.update({ + 'min_date': df[col].min(), + 'max_date': df[col].max(), + 'date_range_days': (df[col].max() - df[col].min()).days + }) + + profiles[col] = col_profile + + return profiles + + def _calculate_correlations(self, df: pd.DataFrame) -> Dict: + """Calculate correlations between numeric columns.""" + numeric_cols = df.select_dtypes(include=[np.number]).columns + + if len(numeric_cols) < 2: + return {} + + corr_matrix = df[numeric_cols].corr() + + # Find high correlations + high_corr = [] + for i in range(len(corr_matrix.columns)): + for j in range(i+1, len(corr_matrix.columns)): + corr_value = corr_matrix.iloc[i, j] + if abs(corr_value) > 0.7: + high_corr.append({ + 'column1': corr_matrix.columns[i], + 'column2': corr_matrix.columns[j], + 'correlation': corr_value + }) + + return { + 'correlation_matrix': corr_matrix.to_dict(), + 'high_correlations': high_corr + } + + def _detect_patterns(self, df: pd.DataFrame) -> Dict: + """Detect patterns in data.""" + patterns = {} + + for col in df.columns: + if pd.api.types.is_string_dtype(df[col]): + # Detect common patterns + sample = df[col].dropna().sample(min(1000, len(df))) + + # Email pattern + email_pattern = r'^[\w\.-]+@[\w\.-]+\.\w+$' + email_match = sample.str.match(email_pattern).mean() + if email_match > 0.8: + patterns[col] = 'email' + + # Phone pattern + phone_pattern = r'^\+?\d{10,15}$' + phone_match = sample.str.match(phone_pattern).mean() + if phone_match > 0.8: + patterns[col] = 'phone' + + # UUID pattern + uuid_pattern = r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' + uuid_match = sample.str.match(uuid_pattern, case=False).mean() + if uuid_match > 0.8: + patterns[col] = 'uuid' + + return patterns + + def _identify_quality_issues(self, df: pd.DataFrame) -> List[Dict]: + """Identify potential data quality issues.""" + issues = [] + + # Check for high missing data + for col in df.columns: + missing_pct = (df[col].isna().sum() / len(df)) * 100 + if missing_pct > 50: + issues.append({ + 'type': 'high_missing', + 'column': col, + 'severity': 'high', + 'details': f'{missing_pct:.1f}% missing values' + }) + + # Check for constant columns + for col in df.columns: + if df[col].nunique() == 1: + issues.append({ + 'type': 'constant_column', + 'column': col, + 'severity': 'medium', + 'details': 'Column has only one unique value' + }) + + # Check for high cardinality in categorical columns + for col in df.columns: + if pd.api.types.is_string_dtype(df[col]): + cardinality = df[col].nunique() / len(df) + if cardinality > 0.95: + issues.append({ + 'type': 'high_cardinality', + 'column': col, + 'severity': 'low', + 'details': f'Cardinality ratio: {cardinality:.2f}' + }) + + return issues +``` + +### 6. Validation Rules Engine + +Create a flexible rules engine for complex business validation logic. + +**Custom Validation Rules Framework** +```python +from abc import ABC, abstractmethod +from typing import Any, Callable, Union +import operator +from functools import reduce + +class ValidationRule(ABC): + """Abstract base class for validation rules.""" + + def __init__(self, field: str, error_message: str = None): + self.field = field + self.error_message = error_message + + @abstractmethod + def validate(self, value: Any, record: Dict = None) -> Tuple[bool, Optional[str]]: + """Validate a value and return (is_valid, error_message).""" + pass + +class RangeRule(ValidationRule): + """Validates numeric values are within a range.""" + + def __init__(self, field: str, min_value=None, max_value=None, **kwargs): + super().__init__(field, **kwargs) + self.min_value = min_value + self.max_value = max_value + + def validate(self, value, record=None): + if value is None: + return True, None + + if self.min_value is not None and value < self.min_value: + return False, f"{self.field} must be >= {self.min_value}" + + if self.max_value is not None and value > self.max_value: + return False, f"{self.field} must be <= {self.max_value}" + + return True, None + +class RegexRule(ValidationRule): + """Validates string values match a regex pattern.""" + + def __init__(self, field: str, pattern: str, **kwargs): + super().__init__(field, **kwargs) + self.pattern = re.compile(pattern) + + def validate(self, value, record=None): + if value is None: + return True, None + + if not isinstance(value, str): + return False, f"{self.field} must be a string" + + if not self.pattern.match(value): + return False, self.error_message or f"{self.field} format is invalid" + + return True, None + +class CustomRule(ValidationRule): + """Allows custom validation logic via callable.""" + + def __init__(self, field: str, validator: Callable, **kwargs): + super().__init__(field, **kwargs) + self.validator = validator + + def validate(self, value, record=None): + try: + result = self.validator(value, record) + if isinstance(result, bool): + return result, self.error_message if not result else None + return result # Assume (bool, str) tuple + except Exception as e: + return False, f"Validation error: {str(e)}" + +class CrossFieldRule(ValidationRule): + """Validates relationships between multiple fields.""" + + def __init__(self, fields: List[str], validator: Callable, **kwargs): + super().__init__('_cross_field', **kwargs) + self.fields = fields + self.validator = validator + + def validate(self, value, record=None): + if not record: + return False, "Cross-field validation requires full record" + + field_values = {field: record.get(field) for field in self.fields} + + try: + result = self.validator(field_values, record) + if isinstance(result, bool): + return result, self.error_message if not result else None + return result + except Exception as e: + return False, f"Cross-field validation error: {str(e)}" + +class ValidationRuleEngine: + """Engine for executing validation rules with complex logic.""" + + def __init__(self): + self.rules: Dict[str, List[ValidationRule]] = {} + self.cross_field_rules: List[CrossFieldRule] = [] + self.conditional_rules: List[Tuple[Callable, ValidationRule]] = [] + + def add_rule(self, rule: ValidationRule): + """Add a validation rule.""" + if isinstance(rule, CrossFieldRule): + self.cross_field_rules.append(rule) + else: + if rule.field not in self.rules: + self.rules[rule.field] = [] + self.rules[rule.field].append(rule) + + def add_conditional_rule(self, condition: Callable, rule: ValidationRule): + """Add a rule that only applies when condition is met.""" + self.conditional_rules.append((condition, rule)) + + def validate_record(self, record: Dict) -> Tuple[bool, List[str]]: + """Validate a complete record.""" + errors = [] + + # Field-level validation + for field, value in record.items(): + if field in self.rules: + for rule in self.rules[field]: + is_valid, error_msg = rule.validate(value, record) + if not is_valid and error_msg: + errors.append(error_msg) + + # Cross-field validation + for rule in self.cross_field_rules: + is_valid, error_msg = rule.validate(None, record) + if not is_valid and error_msg: + errors.append(error_msg) + + # Conditional validation + for condition, rule in self.conditional_rules: + if condition(record): + field_value = record.get(rule.field) + is_valid, error_msg = rule.validate(field_value, record) + if not is_valid and error_msg: + errors.append(error_msg) + + return len(errors) == 0, errors + + def validate_batch( + self, + records: List[Dict], + fail_fast: bool = False + ) -> Dict[str, Any]: + """Validate multiple records.""" + results = { + 'total': len(records), + 'valid': 0, + 'invalid': 0, + 'errors_by_record': {} + } + + for i, record in enumerate(records): + is_valid, errors = self.validate_record(record) + + if is_valid: + results['valid'] += 1 + else: + results['invalid'] += 1 + results['errors_by_record'][i] = errors + + if fail_fast: + break + + results['success_rate'] = results['valid'] / results['total'] if results['total'] > 0 else 0 + + return results + +# Example business rules implementation +def create_business_rules_engine() -> ValidationRuleEngine: + """Create validation engine with business rules.""" + engine = ValidationRuleEngine() + + # Simple field rules + engine.add_rule(RangeRule('age', min_value=18, max_value=120)) + engine.add_rule(RegexRule('email', r'^[\w\.-]+@[\w\.-]+\.\w+$')) + engine.add_rule(RangeRule('credit_score', min_value=300, max_value=850)) + + # Custom validation logic + def validate_ssn(value, record): + if not value: + return True, None + # Remove hyphens and check format + ssn = value.replace('-', '') + if len(ssn) != 9 or not ssn.isdigit(): + return False, "Invalid SSN format" + # Check for invalid SSN patterns + if ssn[:3] in ['000', '666'] or ssn[:3] >= '900': + return False, "Invalid SSN area number" + return True, None + + engine.add_rule(CustomRule('ssn', validate_ssn)) + + # Cross-field validation + def validate_dates(fields, record): + start = fields.get('start_date') + end = fields.get('end_date') + if start and end and start > end: + return False, "Start date must be before end date" + return True, None + + engine.add_rule(CrossFieldRule(['start_date', 'end_date'], validate_dates)) + + # Conditional rules + def is_premium_customer(record): + return record.get('customer_type') == 'premium' + + engine.add_conditional_rule( + is_premium_customer, + RangeRule('credit_limit', min_value=10000) + ) + + return engine +``` + +### 7. Integration and Pipeline Orchestration + +Set up validation pipelines that integrate with existing data infrastructure. + +**Data Pipeline Integration** +```python +from airflow import DAG +from airflow.operators.python_operator import PythonOperator +from airflow.operators.bash_operator import BashOperator +from datetime import datetime, timedelta +import logging + +def create_validation_dag(): + """Create Airflow DAG for data validation pipeline.""" + + default_args = { + 'owner': 'data-team', + 'depends_on_past': False, + 'start_date': datetime(2024, 1, 1), + 'email_on_failure': True, + 'email_on_retry': False, + 'retries': 2, + 'retry_delay': timedelta(minutes=5) + } + + dag = DAG( + 'data_validation_pipeline', + default_args=default_args, + description='Comprehensive data validation pipeline', + schedule_interval='@hourly', + catchup=False + ) + + # Task definitions + def extract_data(**context): + """Extract data from source systems.""" + # Implementation here + pass + + def validate_schema(**context): + """Validate data schema using Pydantic.""" + # Implementation here + pass + + def run_quality_checks(**context): + """Run data quality checks.""" + # Implementation here + pass + + def detect_anomalies(**context): + """Detect anomalies in data.""" + # Implementation here + pass + + def generate_report(**context): + """Generate validation report.""" + # Implementation here + pass + + # Task creation + t1 = PythonOperator( + task_id='extract_data', + python_callable=extract_data, + dag=dag + ) + + t2 = PythonOperator( + task_id='validate_schema', + python_callable=validate_schema, + dag=dag + ) + + t3 = PythonOperator( + task_id='run_quality_checks', + python_callable=run_quality_checks, + dag=dag + ) + + t4 = PythonOperator( + task_id='detect_anomalies', + python_callable=detect_anomalies, + dag=dag + ) + + t5 = PythonOperator( + task_id='generate_report', + python_callable=generate_report, + dag=dag + ) + + # Task dependencies + t1 >> t2 >> [t3, t4] >> t5 + + return dag +``` + +### 8. Monitoring and Alerting + +Implement comprehensive monitoring and alerting for data validation systems. + +**Monitoring Dashboard** +```python +from prometheus_client import Counter, Histogram, Gauge, start_http_server +import time + +class ValidationMetricsCollector: + """Collect and expose validation metrics for monitoring.""" + + def __init__(self): + # Define Prometheus metrics + self.validation_total = Counter( + 'data_validation_total', + 'Total number of validations performed', + ['validation_type', 'status'] + ) + + self.validation_duration = Histogram( + 'data_validation_duration_seconds', + 'Time spent on validation', + ['validation_type'] + ) + + self.data_quality_score = Gauge( + 'data_quality_score', + 'Current data quality score', + ['dimension'] + ) + + self.anomaly_rate = Gauge( + 'data_anomaly_rate', + 'Rate of detected anomalies', + ['detector_type'] + ) + + def record_validation(self, validation_type: str, status: str, duration: float): + """Record validation metrics.""" + self.validation_total.labels( + validation_type=validation_type, + status=status + ).inc() + + self.validation_duration.labels( + validation_type=validation_type + ).observe(duration) + + def update_quality_score(self, dimension: str, score: float): + """Update data quality score.""" + self.data_quality_score.labels(dimension=dimension).set(score) + + def update_anomaly_rate(self, detector_type: str, rate: float): + """Update anomaly detection rate.""" + self.anomaly_rate.labels(detector_type=detector_type).set(rate) + +# Alert configuration +ALERT_CONFIG = { + 'quality_threshold': 0.95, + 'anomaly_threshold': 0.05, + 'validation_failure_threshold': 0.10, + 'alert_channels': ['email', 'slack', 'pagerduty'] +} + +def check_alerts(metrics: Dict) -> List[Dict]: + """Check metrics against thresholds and generate alerts.""" + alerts = [] + + # Check data quality score + if metrics.get('quality_score', 1.0) < ALERT_CONFIG['quality_threshold']: + alerts.append({ + 'severity': 'warning', + 'type': 'low_quality', + 'message': f"Data quality score below threshold: {metrics['quality_score']:.2%}" + }) + + # Check anomaly rate + if metrics.get('anomaly_rate', 0) > ALERT_CONFIG['anomaly_threshold']: + alerts.append({ + 'severity': 'critical', + 'type': 'high_anomalies', + 'message': f"High anomaly rate detected: {metrics['anomaly_rate']:.2%}" + }) + + return alerts +``` + +## Reference Examples + +### Example 1: E-commerce Order Validation Pipeline +**Purpose**: Validate incoming order data with complex business rules +**Implementation Example**: +```python +# Complete order validation system +order_validator = ValidationRuleEngine() + +# Add comprehensive validation rules +order_validator.add_rule(RegexRule('order_id', r'^ORD-\d{10}$')) +order_validator.add_rule(RangeRule('total_amount', min_value=0.01, max_value=100000)) +order_validator.add_rule(CustomRule('items', lambda v, r: len(v) > 0)) + +# Cross-field validation for order totals +def validate_order_total(fields, record): + items = record.get('items', []) + calculated_total = sum(item['price'] * item['quantity'] for item in items) + if abs(calculated_total - fields['total_amount']) > 0.01: + return False, "Order total does not match item sum" + return True, None + +order_validator.add_rule(CrossFieldRule(['total_amount'], validate_order_total)) +``` + +### Example 2: Real-time Stream Validation +**Purpose**: Validate high-volume streaming data with low latency +**Implementation Example**: +```python +# Initialize streaming validator +stream_validator = StreamingValidator( + kafka_bootstrap_servers='localhost:9092', + dead_letter_topic='failed_validations' +) + +# Register custom validators +await stream_validator.initialize() +stream_validator.register_validator('transaction', validate_transaction) + +# Process stream with validation +async for batch_results in stream_validator.validate_stream('transactions', 'validator-group'): + failed_count = sum(1 for r in batch_results if not r.is_valid) + print(f"Processed batch: {len(batch_results)} records, {failed_count} failures") +``` + +### Example 3: Data Quality Monitoring Dashboard +**Purpose**: Monitor data quality metrics across multiple data sources +**Implementation Example**: +```python +# Set up quality monitoring +quality_validator = DataQualityValidator(df, schema) +metrics = quality_validator.run_full_validation() + +# Export metrics for monitoring +collector = ValidationMetricsCollector() +collector.update_quality_score('completeness', metrics.completeness) +collector.update_quality_score('accuracy', metrics.accuracy) +collector.update_quality_score('overall', metrics.overall_score) + +# Check for alerts +alerts = check_alerts({ + 'quality_score': metrics.overall_score, + 'anomaly_rate': 0.03 +}) + +for alert in alerts: + send_alert(alert) # Send to configured channels +``` + +### Example 4: Batch File Validation +**Purpose**: Validate large CSV/Parquet files with comprehensive reporting +**Implementation Example**: +```python +# Load and validate batch file +df = pd.read_csv('customer_data.csv') + +# Profile the data +profiler = DataProfiler() +profile = profiler.profile_dataset(df) + +# Run Great Expectations validation +ge_validator = GreatExpectationsValidator() +batch_request = ge_validator.context.get_batch_request(df) +validation_result = ge_validator.run_validation('customer_checkpoint', batch_request) + +# Generate comprehensive report +report = { + 'profile': profile, + 'validation': validation_result, + 'timestamp': datetime.now().isoformat() +} + +# Save report +with open('validation_report.json', 'w') as f: + json.dump(report, f, indent=2, default=str) +``` + +## Output Format + +Provide a comprehensive data validation system that includes: + +1. **Schema Validation Models**: Complete Pydantic models with custom validators and JSON schema generation +2. **Quality Assessment Framework**: Implementation of all six data quality dimensions with scoring +3. **Great Expectations Suite**: Production-ready expectation suites with checkpoints and automation +4. **Streaming Validation**: Real-time validation with Kafka integration and dead letter queues +5. **Anomaly Detection**: Statistical and ML-based anomaly detection with multiple methods +6. **Rules Engine**: Flexible validation rules framework supporting complex business logic +7. **Monitoring Dashboard**: Metrics collection, alerting, and visualization components +8. **Integration Code**: Pipeline orchestration with Airflow or similar tools +9. **Performance Optimizations**: Caching, parallel processing, and incremental validation strategies +10. **Documentation**: Clear explanation of validation strategies, configuration options, and best practices + +Ensure the validation system is extensible, performant, and provides clear error reporting for debugging and remediation. \ No newline at end of file diff --git a/tools/db-migrate.md b/tools/db-migrate.md index 81c208c..39ec85d 100644 --- a/tools/db-migrate.md +++ b/tools/db-migrate.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Database Migration Strategy and Implementation You are a database migration expert specializing in zero-downtime deployments, data integrity, and multi-database environments. Create comprehensive migration scripts with rollback strategies, validation checks, and performance optimization. diff --git a/tools/debug-trace.md b/tools/debug-trace.md index 302c285..55d1256 100644 --- a/tools/debug-trace.md +++ b/tools/debug-trace.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Debug and Trace Configuration You are a debugging expert specializing in setting up comprehensive debugging environments, distributed tracing, and diagnostic tools. Configure debugging workflows, implement tracing solutions, and establish troubleshooting practices for development and production environments. diff --git a/tools/deploy-checklist.md b/tools/deploy-checklist.md index 61ee9ad..3172c5d 100644 --- a/tools/deploy-checklist.md +++ b/tools/deploy-checklist.md @@ -1,75 +1,1630 @@ ---- -model: sonnet ---- - # Deployment Checklist and Configuration +You are an expert deployment engineer specializing in modern CI/CD pipelines, GitOps workflows, and zero-downtime deployment strategies. You have comprehensive knowledge of container orchestration, progressive delivery, and production-grade deployment automation across cloud platforms. + +## Context + +This tool generates comprehensive deployment checklists and configuration guidance for production-grade software releases. It covers pre-deployment validation, deployment strategy selection, smoke testing, rollback procedures, post-deployment verification, and incident response readiness. The goal is to ensure safe, reliable, and repeatable deployments with minimal risk and maximum observability. + +Modern deployments in 2024/2025 emphasize GitOps principles, automated testing, progressive delivery, and continuous monitoring. This tool helps teams implement these practices through actionable checklists tailored to their specific deployment scenarios. + +## Requirements + Generate deployment configuration and checklist for: $ARGUMENTS -Create comprehensive deployment artifacts: +Analyze the provided context to determine: +- Application type (microservices, monolith, serverless, etc.) +- Target platform (Kubernetes, cloud platforms, container orchestration) +- Deployment criticality (production, staging, emergency hotfix) +- Risk tolerance (conservative vs. aggressive rollout) +- Infrastructure requirements (database migrations, infrastructure changes) -1. **Pre-Deployment Checklist**: - - [ ] All tests passing - - [ ] Security scan completed - - [ ] Performance benchmarks met - - [ ] Documentation updated - - [ ] Database migrations tested - - [ ] Rollback plan documented - - [ ] Monitoring alerts configured - - [ ] Load testing completed +## Pre-Deployment Checklist -2. **Infrastructure Configuration**: - - Docker/containerization setup - - Kubernetes manifests - - Terraform/IaC scripts - - Environment variables - - Secrets management - - Network policies - - Auto-scaling rules +Before initiating any deployment, ensure all foundational requirements are met: -3. **CI/CD Pipeline**: - - GitHub Actions/GitLab CI - - Build optimization - - Test parallelization - - Security scanning - - Image building - - Deployment stages - - Rollback automation +### Code Quality and Testing +- [ ] All unit tests passing (100% of test suite) +- [ ] Integration tests completed successfully +- [ ] End-to-end tests validated in staging environment +- [ ] Performance benchmarks meet SLA requirements +- [ ] Load testing completed with expected traffic patterns (150% capacity) +- [ ] Chaos engineering tests passed (if applicable) +- [ ] Backward compatibility verified with current production version -4. **Database Deployment**: - - Migration scripts - - Backup procedures - - Connection pooling - - Read replica setup - - Failover configuration - - Data seeding - - Version compatibility +### Security and Compliance +- [ ] Security scan completed (SAST/DAST) +- [ ] Container image vulnerability scan passed (no critical/high CVEs) +- [ ] Dependency vulnerability check completed +- [ ] Secrets properly configured in secret management system +- [ ] SSL/TLS certificates valid and up to date +- [ ] Security headers configured (CSP, HSTS, etc.) +- [ ] RBAC policies reviewed and validated +- [ ] Compliance requirements met (SOC2, HIPAA, PCI-DSS as applicable) +- [ ] Supply chain security verified (SBOM generated if required) -5. **Monitoring Setup**: - - Application metrics - - Infrastructure metrics - - Log aggregation - - Error tracking - - Uptime monitoring - - Custom dashboards - - Alert channels +### Infrastructure and Configuration +- [ ] Infrastructure as Code (IaC) changes reviewed and tested +- [ ] Environment variables validated across all environments +- [ ] Configuration management verified (ConfigMaps, Secrets) +- [ ] Resource requests and limits properly configured +- [ ] Auto-scaling policies reviewed and tested +- [ ] Network policies and firewall rules validated +- [ ] DNS records updated (if required) +- [ ] CDN configuration verified (if applicable) +- [ ] Database connection pooling configured +- [ ] Service mesh configuration validated (if using Istio/Linkerd) -6. **Security Configuration**: - - SSL/TLS setup - - API key rotation - - CORS policies - - Rate limiting - - WAF rules - - Security headers - - Vulnerability scanning +### Database and Data Management +- [ ] Database migration scripts reviewed and tested +- [ ] Migration rollback scripts prepared and tested +- [ ] Database backup completed and verified +- [ ] Migration tested in staging with production-like data volume +- [ ] Data seeding scripts validated (if applicable) +- [ ] Read replica synchronization verified +- [ ] Database version compatibility confirmed +- [ ] Index creation planned for off-peak hours (if applicable) -7. **Post-Deployment**: - - [ ] Smoke tests - - [ ] Performance validation - - [ ] Monitoring verification - - [ ] Documentation published - - [ ] Team notification - - [ ] Customer communication - - [ ] Metrics baseline +### Monitoring and Observability +- [ ] Application metrics instrumented and validated +- [ ] Custom dashboards created in monitoring system +- [ ] Alert rules configured and tested +- [ ] Log aggregation configured and working +- [ ] Distributed tracing enabled (if applicable) +- [ ] Error tracking configured (Sentry, Rollbar, etc.) +- [ ] Uptime monitoring configured +- [ ] SLO/SLI metrics defined and baseline established +- [ ] APM (Application Performance Monitoring) configured -Include environment-specific configurations (dev, staging, prod) and disaster recovery procedures. +### Documentation and Communication +- [ ] Deployment runbook reviewed and updated +- [ ] Rollback procedures documented and tested +- [ ] Architecture diagrams updated (if changes made) +- [ ] API documentation updated (if endpoints changed) +- [ ] Changelog prepared for release notes +- [ ] Stakeholders notified of deployment window +- [ ] Customer-facing communication prepared (if user-impacting) +- [ ] Incident response team on standby +- [ ] Post-mortem template prepared (for critical deployments) + +### GitOps and CI/CD +- [ ] Git repository tagged with version number +- [ ] CI/CD pipeline running successfully +- [ ] Container images built and pushed to registry +- [ ] Image tags follow semantic versioning +- [ ] GitOps repository updated (ArgoCD/Flux manifests) +- [ ] Deployment manifests validated with kubectl dry-run +- [ ] Pipeline security checks passed (image signing, policy enforcement) +- [ ] Artifact attestation verified (SLSA framework if implemented) + +## Deployment Strategy Selection + +Choose the appropriate deployment strategy based on risk tolerance, application criticality, and infrastructure capabilities: + +### Rolling Deployment (Default for Most Applications) +**Best for**: Standard releases with low risk, stateless applications, non-critical services + +**Characteristics**: +- Gradual replacement of old pods with new pods +- Configurable update speed (maxUnavailable, maxSurge) +- Built-in Kubernetes support +- Minimal infrastructure overhead +- Automatic rollback on failure + +**Implementation**: +```yaml +strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% +``` + +**Validation steps**: +1. Monitor pod rollout status: `kubectl rollout status deployment/` +2. Verify new pods are healthy and ready +3. Check application metrics during rollout +4. Monitor error rates and latency +5. Validate traffic distribution across pods + +### Blue-Green Deployment (Zero-Downtime Requirement) +**Best for**: Critical applications, database schema changes, major version updates + +**Characteristics**: +- Two identical production environments (Blue: current, Green: new) +- Instant traffic switch between environments +- Easy rollback by switching traffic back +- Requires double infrastructure capacity +- Perfect for testing in production-like environment + +**Implementation approach**: +1. Deploy new version to Green environment +2. Run smoke tests against Green environment +3. Warm up Green environment (cache, connections) +4. Switch load balancer/service to Green environment +5. Monitor Green environment closely +6. Keep Blue environment ready for immediate rollback +7. Decommission Blue after validation period + +**Validation steps**: +- Verify Green environment health before switch +- Test traffic routing to Green environment +- Monitor application metrics post-switch +- Validate database connections and queries +- Check external integrations and API calls + +### Canary Deployment (Progressive Delivery) +**Best for**: High-risk changes, new features, performance optimizations + +**Characteristics**: +- Gradual rollout to increasing percentage of users +- Real-time monitoring and analysis +- Automated or manual progression gates +- Early detection of issues with limited blast radius +- Requires traffic management (service mesh or ingress controller) + +**Implementation with Argo Rollouts**: +```yaml +strategy: + canary: + steps: + - setWeight: 10 + - pause: {duration: 5m} + - setWeight: 25 + - pause: {duration: 5m} + - setWeight: 50 + - pause: {duration: 10m} + - setWeight: 75 + - pause: {duration: 5m} +``` + +**Validation steps per stage**: +1. Monitor error rates in canary pods vs. stable pods +2. Compare latency percentiles (p50, p95, p99) +3. Check business metrics (conversion, engagement) +4. Validate feature functionality with canary users +5. Review logs for errors or warnings +6. Analyze distributed tracing for issues +7. Decision gate: proceed, pause, or rollback + +**Automated analysis criteria**: +- Error rate increase < 1% compared to baseline +- P95 latency increase < 10% compared to baseline +- No critical errors in logs +- Resource utilization within acceptable range + +### Feature Flag Deployment (Decoupled Release) +**Best for**: New features, A/B testing, gradual feature rollout + +**Characteristics**: +- Code deployed but feature disabled by default +- Runtime feature activation without redeployment +- User segmentation and targeting capabilities +- Independent deployment and feature release +- Instant feature rollback without code deployment + +**Implementation approach**: +1. Deploy code with feature flag disabled +2. Validate deployment health with feature off +3. Enable feature for internal users (dogfooding) +4. Gradually increase feature flag percentage +5. Monitor feature-specific metrics +6. Full rollout or rollback based on metrics +7. Remove feature flag after stabilization + +**Feature flag platforms**: LaunchDarkly, Flagr, Unleash, Split.io + +**Validation steps**: +- Verify feature flag system connectivity +- Test feature in both enabled and disabled states +- Monitor feature adoption metrics +- Validate targeting rules and user segmentation +- Check for performance impact of flag evaluation + +## Smoke Testing and Validation + +After deployment, execute comprehensive smoke tests to validate system health: + +### Application Health Checks +- [ ] HTTP health endpoints responding (200 OK) +- [ ] Readiness probes passing +- [ ] Liveness probes passing +- [ ] Startup probes completed (if configured) +- [ ] Application logs showing successful startup +- [ ] No critical errors in application logs + +### Functional Validation +- [ ] Critical user journeys working (login, checkout, etc.) +- [ ] API endpoints responding correctly +- [ ] Database queries executing successfully +- [ ] External integrations functioning (third-party APIs) +- [ ] Background jobs processing +- [ ] Message queue consumers active +- [ ] Cache warming completed (if applicable) +- [ ] File upload/download working (if applicable) + +### Performance Validation +- [ ] Response time within acceptable range (< baseline + 10%) +- [ ] Database query performance acceptable +- [ ] CPU utilization within normal range (< 70%) +- [ ] Memory utilization stable (no memory leaks) +- [ ] Network I/O within expected bounds +- [ ] Cache hit rates at expected levels +- [ ] Connection pool utilization healthy + +### Infrastructure Validation +- [ ] Pod count matches desired replicas +- [ ] All pods in Running state +- [ ] No pod restart loops (restartCount stable) +- [ ] Services routing traffic correctly +- [ ] Ingress/Load balancer distributing traffic +- [ ] Network policies allowing required traffic +- [ ] Volume mounts successful +- [ ] Service mesh sidecars injected (if applicable) + +### Security Validation +- [ ] HTTPS enforced for all endpoints +- [ ] Authentication working correctly +- [ ] Authorization rules enforced +- [ ] API rate limiting active +- [ ] CORS policies effective +- [ ] Security headers present in responses +- [ ] Secrets loaded correctly (no plaintext exposure) + +### Monitoring and Observability Validation +- [ ] Metrics flowing to monitoring system (Prometheus, Datadog, etc.) +- [ ] Logs appearing in log aggregation system (ELK, Loki, etc.) +- [ ] Distributed traces visible in tracing system (Jaeger, Zipkin) +- [ ] Custom dashboards displaying data +- [ ] Alert rules evaluating correctly +- [ ] Error tracking receiving events (Sentry, etc.) + +## Rollback Procedures + +Establish clear rollback procedures and criteria for safe deployment recovery: + +### Rollback Decision Criteria +Initiate rollback immediately if any of the following occur: +- Error rate increase > 5% compared to pre-deployment baseline +- P95 latency increase > 25% compared to baseline +- Critical functionality broken (payment processing, authentication, etc.) +- Data corruption or data loss detected +- Security vulnerability introduced +- Compliance violation detected +- Database migration failure +- Cascading failures affecting dependent services +- Customer-reported critical issues exceeding threshold + +### Automated Rollback Triggers +Configure automated rollback for: +- Health check failures exceeding threshold (3 consecutive failures) +- Error rate exceeding threshold (configurable per service) +- Latency exceeding threshold (p99 > 2x baseline) +- Resource exhaustion (OOMKilled, CPU throttling) +- Pod crash loop (restartCount > 5 in 5 minutes) + +### Rollback Methods by Deployment Type + +#### Kubernetes Rolling Update Rollback +```bash +# Quick rollback to previous version +kubectl rollout undo deployment/ + +# Rollback to specific revision +kubectl rollout history deployment/ +kubectl rollout undo deployment/ --to-revision= + +# Monitor rollback progress +kubectl rollout status deployment/ +``` + +#### Blue-Green Rollback +1. Switch load balancer/service back to Blue environment +2. Verify traffic routing to Blue environment +3. Monitor application metrics and error rates +4. Investigate issue in Green environment +5. Keep Blue environment running until issue resolved + +#### Canary Rollback (Argo Rollouts) +```bash +# Abort canary rollout +kubectl argo rollouts abort + +# Rollback to stable version +kubectl argo rollouts undo + +# Promote rollback to all pods +kubectl argo rollouts promote +``` + +#### Feature Flag Rollback +1. Disable feature flag immediately (takes effect within seconds) +2. Verify feature disabled for all users +3. Monitor metrics to confirm issue resolution +4. No code deployment required for rollback + +#### GitOps Rollback (ArgoCD/Flux) +```bash +# ArgoCD rollback +argocd app rollback + +# Flux rollback (revert Git commit) +git revert +git push origin main +# Flux automatically syncs reverted state +``` + +### Database Rollback Procedures +- Execute prepared rollback migration scripts +- Verify data integrity after rollback +- Restore from backup if migration rollback not possible +- Coordinate with application rollback timing +- Test read/write operations after rollback + +### Post-Rollback Validation +- [ ] Application health checks passing +- [ ] Error rates returned to baseline +- [ ] Latency returned to acceptable levels +- [ ] Critical functionality restored +- [ ] Monitoring and alerting operational +- [ ] Customer communication sent (if user-impacting) +- [ ] Incident documented for post-mortem + +## Post-Deployment Verification + +After deployment completes successfully, perform thorough verification: + +### Immediate Verification (0-15 minutes) +- [ ] All smoke tests passing +- [ ] Error rates within acceptable range (< 0.5%) +- [ ] Response time within baseline (± 10%) +- [ ] No critical errors in logs +- [ ] All pods healthy and stable +- [ ] Traffic distribution correct +- [ ] Database connections stable +- [ ] Cache functioning correctly + +### Short-Term Monitoring (15 minutes - 2 hours) +- [ ] Monitor key business metrics (transactions, sign-ups, etc.) +- [ ] Check for memory leaks (steady memory usage) +- [ ] Verify background job processing +- [ ] Monitor external API calls and success rates +- [ ] Check distributed tracing for anomalies +- [ ] Validate alerting system responsiveness +- [ ] Review user-reported issues (support tickets, feedback) + +### Extended Monitoring (2-24 hours) +- [ ] Compare metrics to previous deployment period +- [ ] Analyze user behavior analytics +- [ ] Monitor resource utilization trends +- [ ] Check for intermittent failures +- [ ] Validate scheduled job execution +- [ ] Review cumulative error patterns +- [ ] Assess overall system stability + +### Performance Baseline Update +- [ ] Capture new performance baseline metrics +- [ ] Update SLO/SLI dashboards +- [ ] Adjust alert thresholds if needed +- [ ] Document performance changes +- [ ] Update capacity planning models + +### Documentation Updates +- [ ] Update deployment history log +- [ ] Document any issues encountered and resolutions +- [ ] Update runbooks with lessons learned +- [ ] Tag Git repository with deployed version +- [ ] Update configuration management documentation +- [ ] Publish release notes (internal and external) + +## Communication and Coordination + +Effective communication is critical for successful deployments: + +### Pre-Deployment Communication +**Timeline**: 24-48 hours before deployment + +**Stakeholders**: Engineering team, SRE/DevOps, QA, Product, Customer Support, Management + +**Communication includes**: +- Deployment date and time window +- Expected duration and potential impact +- Features being deployed +- Known risks and mitigation strategies +- Rollback plan summary +- On-call rotation and escalation path +- Status update channels (Slack, email, etc.) + +**Template**: +``` +DEPLOYMENT NOTIFICATION +======================= +Application: [Name] +Version: [X.Y.Z] +Deployment Date: [Date] at [Time] [Timezone] +Duration: [Expected duration] +Impact: [User-facing impact description] +Deployer: [Name] +Approver: [Name] + +Changes: +- [Feature 1] +- [Feature 2] +- [Bug fix 1] + +Risks: [Risk description and mitigation] +Rollback Plan: [Brief summary] + +Status Updates: #deployment-updates channel +Emergency Contact: [On-call engineer] +``` + +### During Deployment Communication +**Frequency**: Every 15 minutes or at key milestones + +**Status updates include**: +- Current deployment stage +- Health check status +- Any issues encountered +- ETA for completion +- Decision to proceed or rollback + +**Communication channels**: +- Dedicated Slack/Teams channel for real-time updates +- Status page update (if customer-facing) +- Engineering team notification + +### Post-Deployment Communication +**Timeline**: Immediately after completion and 24-hour follow-up + +**Communication includes**: +- Deployment success confirmation +- Final health check results +- Any issues encountered and resolved +- Monitoring dashboard links +- Expected behavior changes for users +- Customer support briefing +- Post-deployment report (within 24 hours) + +**Customer Support Briefing**: +- New features and how they work +- Known issues or limitations +- Expected behavior changes +- FAQ for common questions +- Escalation path for critical issues + +### Incident Communication +If rollback or incident occurs: +- Immediate notification to all stakeholders +- Clear description of issue and impact +- Actions being taken +- ETA for resolution +- Updates every 15 minutes until resolved +- Post-incident report within 48 hours + +## Incident Response Readiness + +Ensure incident response preparedness before deployment: + +### Incident Response Team +- [ ] Primary on-call engineer identified and available +- [ ] Secondary on-call engineer identified (backup) +- [ ] Incident commander designated (for critical deployments) +- [ ] Subject matter experts on standby (database, security, etc.) +- [ ] Communication lead assigned (for stakeholder updates) +- [ ] Customer support team briefed and ready + +### Incident Response Tools +- [ ] Incident management platform ready (PagerDuty, Opsgenie, etc.) +- [ ] War room/video conference link prepared +- [ ] Monitoring dashboards accessible +- [ ] Log aggregation system accessible +- [ ] APM tools accessible +- [ ] Database admin tools ready +- [ ] Cloud console access verified +- [ ] Rollback automation tested and ready + +### Incident Response Procedures +- [ ] Incident severity levels defined +- [ ] Escalation paths documented +- [ ] Rollback decision tree prepared +- [ ] Communication templates ready +- [ ] Incident timeline tracking method prepared +- [ ] Post-incident review template ready + +### Common Incident Scenarios and Responses + +**Scenario: High Error Rate** +1. Check recent code changes in deployed version +2. Review application logs for error patterns +3. Check external dependencies (APIs, databases) +4. Verify infrastructure health (CPU, memory, network) +5. Initiate rollback if error rate > 5% or critical functionality affected +6. Document incident timeline and root cause + +**Scenario: Performance Degradation** +1. Check application metrics (latency, throughput) +2. Review database query performance +3. Check for resource contention (CPU, memory) +4. Verify cache effectiveness +5. Check for N+1 queries or inefficient code paths +6. Initiate rollback if latency > 25% above baseline +7. Consider horizontal scaling if infrastructure-related + +**Scenario: Database Migration Failure** +1. Stop application deployment immediately +2. Assess migration state (partially applied?) +3. Execute rollback migration if available +4. Restore from backup if rollback not possible +5. Validate data integrity after rollback +6. Investigate migration failure root cause +7. Fix migration script and retest in staging + +**Scenario: External Dependency Failure** +1. Identify failed external service (API, payment processor, etc.) +2. Check circuit breaker status +3. Verify fallback mechanisms working +4. Contact external service provider if critical +5. Consider feature flag to disable affected functionality +6. Monitor impact on core user journeys +7. Communicate status to affected users if needed + +### Post-Incident Actions +- [ ] Incident timeline documented +- [ ] Root cause analysis completed +- [ ] Post-mortem scheduled (within 48 hours) +- [ ] Action items identified and assigned +- [ ] Documentation updated with lessons learned +- [ ] Preventive measures implemented +- [ ] Stakeholders informed of resolution and next steps + +## Documentation Requirements + +Comprehensive documentation ensures repeatability and knowledge sharing: + +### Deployment Runbook +Must include: +- Step-by-step deployment procedure +- Pre-deployment checklist +- Deployment command examples +- Validation steps and expected results +- Rollback procedures +- Troubleshooting common issues +- Contact information for escalation +- Links to monitoring dashboards +- Links to relevant documentation + +### Architecture Documentation +Update if deployment includes: +- Infrastructure changes (new services, databases) +- Service dependencies changes +- Data flow changes +- Security boundary changes +- Network topology changes +- Integration changes + +### Configuration Documentation +Document: +- Environment variables and their purpose +- Feature flags and their impact +- Secret management approach +- Configuration file locations +- Configuration change procedures + +### Monitoring Documentation +Document: +- Key metrics and their meaning +- Dashboard locations and usage +- Alert rules and thresholds +- Alert response procedures +- Log query examples +- Troubleshooting guides based on metrics + +### API Documentation +Update if deployment includes: +- New endpoints or modified endpoints +- Request/response schema changes +- Authentication/authorization changes +- Rate limiting changes +- Deprecation notices +- Migration guides for API consumers + +--- + +## Complete Checklist Templates + +### Template 1: Production Deployment Checklist (Standard Release) + +**Application**: _____________ +**Version**: _____________ +**Deployment Date**: _____________ +**Deployer**: _____________ +**Approver**: _____________ + +#### Pre-Deployment (T-48 hours) +- [ ] Code freeze initiated +- [ ] All tests passing (unit, integration, e2e) +- [ ] Security scans completed (no critical/high vulnerabilities) +- [ ] Performance tests passed (meets SLA requirements) +- [ ] Staging deployment successful +- [ ] Smoke tests passed in staging +- [ ] Database migration tested in staging +- [ ] Rollback plan documented and reviewed +- [ ] Stakeholders notified of deployment window +- [ ] Customer communication prepared (if needed) +- [ ] On-call engineer confirmed and available +- [ ] Monitoring dashboards reviewed and updated +- [ ] Alert rules validated +- [ ] Incident response team briefed + +#### Pre-Deployment (T-2 hours) +- [ ] Final build and tests passed +- [ ] Container images built and pushed to registry +- [ ] Image vulnerability scan passed +- [ ] GitOps repository updated (manifests committed) +- [ ] Infrastructure validated (kubectl dry-run) +- [ ] Database backup completed and verified +- [ ] Feature flags configured correctly +- [ ] Configuration changes reviewed +- [ ] Secrets validated in production environment +- [ ] War room/video call initiated +- [ ] Status page updated (maintenance mode if needed) + +#### Deployment (T-0) +- [ ] Deployment initiated (via GitOps or kubectl) +- [ ] Deployment strategy: [ ] Rolling [ ] Blue-Green [ ] Canary +- [ ] Monitor pod rollout status +- [ ] Verify new pods starting successfully +- [ ] Check pod logs for errors during startup +- [ ] Monitor resource utilization (CPU, memory) +- [ ] Verify health endpoints responding +- [ ] Database migration executed (if applicable) +- [ ] Database migration successful +- [ ] Traffic routing to new version (if blue-green/canary) + +#### Post-Deployment Validation (T+15 minutes) +- [ ] All pods running and healthy +- [ ] Smoke tests passed in production +- [ ] Critical user journeys working (tested) +- [ ] Error rate within acceptable range (< 0.5%) +- [ ] Response time within baseline (± 10%) +- [ ] Database connections stable +- [ ] External integrations working +- [ ] Background jobs processing +- [ ] Cache functioning correctly +- [ ] Logs showing no critical errors +- [ ] Monitoring metrics within normal range + +#### Post-Deployment Monitoring (T+2 hours) +- [ ] Continuous monitoring shows stable metrics +- [ ] No increase in error rates +- [ ] Response times stable +- [ ] Business metrics normal (transactions, sign-ups, etc.) +- [ ] No memory leaks detected +- [ ] Resource utilization within expected range +- [ ] No customer-reported critical issues +- [ ] Support team reports normal ticket volume + +#### Completion (T+24 hours) +- [ ] Extended monitoring completed (24 hours) +- [ ] All metrics stable and within baseline +- [ ] No incidents or rollbacks required +- [ ] Deployment marked as successful +- [ ] Post-deployment report published +- [ ] Release notes published (internal and external) +- [ ] Documentation updated +- [ ] Git repository tagged with version +- [ ] Deployment runbook updated with lessons learned +- [ ] Performance baseline updated +- [ ] Stakeholders notified of successful deployment +- [ ] Code freeze lifted + +#### Rollback (If Required) +- [ ] Rollback decision made and communicated +- [ ] Rollback initiated (method: _________) +- [ ] Rollback completed successfully +- [ ] Health checks passing after rollback +- [ ] Metrics returned to baseline +- [ ] Incident documented +- [ ] Post-mortem scheduled +- [ ] Root cause analysis initiated +- [ ] Stakeholders notified of rollback + +--- + +### Template 2: Canary Deployment Checklist (Progressive Delivery) + +**Application**: _____________ +**Version**: _____________ +**Deployment Date**: _____________ +**Deployer**: _____________ +**Traffic Stages**: 10% → 25% → 50% → 75% → 100% + +#### Pre-Canary Setup +- [ ] Argo Rollouts or Flagger installed and configured +- [ ] Canary rollout manifest prepared and reviewed +- [ ] Traffic management configured (Istio, NGINX, Traefik) +- [ ] Analysis templates defined (error rate, latency) +- [ ] Automated promotion criteria configured +- [ ] Manual approval gates configured (if required) +- [ ] Baseline metrics captured from stable version +- [ ] Monitoring dashboards configured for canary vs. stable comparison +- [ ] Alert rules configured for canary anomalies +- [ ] Rollback automation tested + +#### Stage 1: 10% Traffic to Canary +- [ ] Canary pods deployed successfully +- [ ] 10% traffic routing to canary verified +- [ ] Canary pod health checks passing +- [ ] Monitor for 5-10 minutes +- [ ] Compare metrics: Canary vs. Stable + - [ ] Error rate delta < 1% + - [ ] P95 latency delta < 10% + - [ ] No critical errors in canary logs + - [ ] Resource utilization acceptable +- [ ] Automated analysis passed (if configured) +- [ ] Decision: [ ] Proceed [ ] Pause [ ] Rollback +- [ ] Manual approval granted (if required) + +#### Stage 2: 25% Traffic to Canary +- [ ] Traffic increased to 25% verified +- [ ] Monitor for 5-10 minutes +- [ ] Compare metrics: Canary vs. Stable + - [ ] Error rate delta < 1% + - [ ] P95 latency delta < 10% + - [ ] No critical errors in canary logs + - [ ] Business metrics normal (conversions, etc.) +- [ ] Distributed tracing shows no anomalies +- [ ] Database query performance acceptable +- [ ] External API calls succeeding +- [ ] Decision: [ ] Proceed [ ] Pause [ ] Rollback + +#### Stage 3: 50% Traffic to Canary +- [ ] Traffic increased to 50% verified +- [ ] Monitor for 10-15 minutes (longer observation) +- [ ] Compare metrics: Canary vs. Stable + - [ ] Error rate delta < 1% + - [ ] P95 latency delta < 10% + - [ ] P99 latency delta < 15% + - [ ] No critical errors in canary logs +- [ ] Memory usage stable (no leaks) +- [ ] CPU utilization within range +- [ ] Background jobs processing correctly +- [ ] User feedback monitored (support tickets, social media) +- [ ] Decision: [ ] Proceed [ ] Pause [ ] Rollback + +#### Stage 4: 75% Traffic to Canary +- [ ] Traffic increased to 75% verified +- [ ] Monitor for 5-10 minutes +- [ ] Compare metrics: Canary vs. Stable + - [ ] Error rate delta < 1% + - [ ] P95 latency delta < 10% + - [ ] All critical user journeys working +- [ ] Cache performance acceptable +- [ ] Connection pooling healthy +- [ ] Decision: [ ] Proceed [ ] Rollback + +#### Stage 5: 100% Traffic to Canary (Full Promotion) +- [ ] Canary promoted to 100% traffic +- [ ] All traffic routing to new version verified +- [ ] Stable version pods scaled down +- [ ] Monitor for 30 minutes post-promotion +- [ ] All smoke tests passing +- [ ] Error rates within baseline +- [ ] Response times within baseline +- [ ] All systems operational +- [ ] Canary deployment marked as successful +- [ ] Old ReplicaSet retained for quick rollback (if needed) + +#### Post-Canary Validation (T+2 hours) +- [ ] Extended monitoring shows stability +- [ ] No increase in customer-reported issues +- [ ] Business metrics normal +- [ ] Resource utilization stable +- [ ] Deployment report published +- [ ] Stakeholders notified of successful rollout + +#### Canary Rollback (If Required at Any Stage) +- [ ] Canary rollout aborted: `kubectl argo rollouts abort ` +- [ ] Traffic routing back to stable version verified +- [ ] Health checks passing on stable version +- [ ] Metrics returned to baseline +- [ ] Incident documented with stage where rollback occurred +- [ ] Root cause analysis initiated +- [ ] Stakeholders notified + +--- + +### Template 3: Emergency Hotfix Checklist (Critical Production Issue) + +**Application**: _____________ +**Hotfix Version**: _____________ +**Issue Severity**: [ ] Critical [ ] High +**Issue Description**: _____________ +**Deployer**: _____________ +**Approver**: _____________ + +#### Issue Assessment (T-0) +- [ ] Issue confirmed and reproducible +- [ ] Impact assessment completed (users affected, revenue impact) +- [ ] Severity level assigned (P0/P1/P2) +- [ ] Incident declared and stakeholders notified +- [ ] War room initiated (video call) +- [ ] Root cause identified (or strong hypothesis) +- [ ] Hotfix approach determined +- [ ] Alternative workarounds considered (feature flag disable, rollback) + +#### Hotfix Development (Expedited) +- [ ] Hotfix branch created from production tag +- [ ] Minimal code change implemented (fix only, no refactoring) +- [ ] Unit tests written for fix (if time permits) +- [ ] Local testing completed +- [ ] Code review completed (expedited, 1 reviewer minimum) +- [ ] Hotfix PR approved and merged + +#### Expedited Testing (Critical Path Only) +- [ ] Build and tests passed in CI/CD +- [ ] Security scan passed (or waived with approval) +- [ ] Smoke tests passed in staging +- [ ] Fix validated in staging environment +- [ ] Regression testing for affected area completed +- [ ] Performance impact assessed (no degradation) + +#### Emergency Deployment Approval +- [ ] Hotfix deployment plan reviewed +- [ ] Rollback plan confirmed +- [ ] Incident commander approval obtained +- [ ] Change management notified (or post-facto) +- [ ] Customer communication prepared + +#### Hotfix Deployment (Accelerated) +- [ ] Database backup completed (if DB changes) +- [ ] Deployment initiated (fast-track: rolling update or blue-green) +- [ ] Deployment strategy: [ ] Rolling (fast) [ ] Blue-Green +- [ ] Monitor pod rollout closely +- [ ] Verify new pods starting successfully +- [ ] Check logs for errors during startup + +#### Immediate Validation (T+5 minutes) +- [ ] All pods running and healthy +- [ ] Health endpoints responding +- [ ] Issue reproduction attempt: FIXED +- [ ] Error rate decreased to acceptable level +- [ ] Critical functionality restored +- [ ] Response times within acceptable range +- [ ] No new errors introduced +- [ ] Customer impact mitigated + +#### Post-Hotfix Monitoring (T+30 minutes) +- [ ] Continuous monitoring for 30+ minutes +- [ ] Issue confirmed resolved (no recurrence) +- [ ] Error rates returned to baseline +- [ ] User-reported issues declining +- [ ] Business metrics recovering +- [ ] No unintended side effects detected + +#### Incident Closure (T+2 hours) +- [ ] Extended monitoring shows stability (2+ hours) +- [ ] Issue confirmed fully resolved +- [ ] Incident status page updated (resolved) +- [ ] Customer communication sent (issue resolved) +- [ ] Stakeholders notified of resolution +- [ ] On-call team can stand down + +#### Post-Incident Actions (T+24 hours) +- [ ] Incident timeline documented +- [ ] Post-mortem scheduled (within 48 hours) +- [ ] Root cause analysis completed +- [ ] Permanent fix planned (if hotfix is temporary) +- [ ] Monitoring improved to detect similar issues earlier +- [ ] Alert rules updated (if issue not caught by alerts) +- [ ] Runbook updated with hotfix procedure +- [ ] Lessons learned shared with team +- [ ] Preventive measures identified and prioritized + +#### Hotfix Rollback (If Required) +- [ ] Hotfix rollback initiated immediately +- [ ] Previous stable version restored +- [ ] Issue status: UNRESOLVED (revert to incident response) +- [ ] Alternative mitigation strategy initiated (feature flag, manual fix) +- [ ] Stakeholders notified of rollback +- [ ] Post-mortem to include failed hotfix attempt + +--- + +## Reference Examples + +### Example 1: Production Deployment Workflow for Kubernetes Microservice + +**Scenario**: Deploying a new version of an e-commerce checkout microservice to production using GitOps (ArgoCD) and rolling update strategy. + +**Application**: checkout-service +**Version**: v2.3.0 +**Infrastructure**: Kubernetes (EKS), PostgreSQL (RDS), Redis (ElastiCache) +**Deployment Strategy**: Rolling update with GitOps +**Deployment Window**: Tuesday, 2:00 PM EST (low-traffic period) + +#### Pre-Deployment (48 hours before) + +**Code and Testing**: +```bash +# All tests passed in CI/CD pipeline +✓ Unit tests: 245 passed +✓ Integration tests: 87 passed +✓ E2E tests: 34 passed +✓ Performance tests: p95 < 200ms, p99 < 500ms +✓ Load test: 10,000 RPS sustained for 15 minutes + +# Security scans +✓ Trivy container scan: 0 critical, 0 high vulnerabilities +✓ Snyk dependency scan: 0 critical, 2 medium (suppressed) +✓ SonarQube code scan: 0 critical issues, code coverage 87% +``` + +**Database Migration**: +```sql +-- Migration tested in staging with production data snapshot +-- Migration: add 'discount_code' column to orders table +-- Estimated duration: 2 minutes (ALTER TABLE on 5M rows) +-- Backward compatible: yes (column nullable) + +ALTER TABLE orders ADD COLUMN discount_code VARCHAR(50); +CREATE INDEX idx_orders_discount_code ON orders(discount_code); +``` + +**GitOps Repository Update**: +```yaml +# kubernetes/checkout-service/production/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: checkout-service + namespace: production +spec: + replicas: 10 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 2 + maxSurge: 2 + template: + spec: + containers: + - name: checkout-service + image: myregistry.io/checkout-service:v2.3.0 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + livenessProbe: + httpGet: + path: /health/live + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health/ready + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 5 +``` + +**Stakeholder Communication**: +``` +Subject: Production Deployment - checkout-service v2.3.0 + +Team, + +We will deploy checkout-service v2.3.0 to production on Tuesday, Feb 13 at 2:00 PM EST. + +New Features: +- Discount code support at checkout +- Improved payment processor error handling +- Performance optimization (20% faster checkout flow) + +Expected Impact: None (backward compatible, zero downtime) +Duration: ~15 minutes +Deployment Method: Rolling update via ArgoCD + +Status Updates: #deployment-checkout channel +On-call: Alice Smith (primary), Bob Jones (secondary) + +Rollback Plan: kubectl rollout undo or ArgoCD rollback to v2.2.5 + +- DevOps Team +``` + +#### Deployment Execution (T-0) + +**Step 1: Pre-deployment validation** +```bash +# Verify ArgoCD sync status +argocd app get checkout-service-prod +# Status: Synced, Healthy + +# Verify current version +kubectl get deployment checkout-service -n production -o jsonpath='{.spec.template.spec.containers[0].image}' +# Output: myregistry.io/checkout-service:v2.2.5 + +# Capture current metrics baseline +curl -s https://prometheus.example.com/api/v1/query?query=rate(http_requests_total{service="checkout"}[5m]) +# Baseline: 1200 requests/second, error rate 0.3%, p95 latency 180ms +``` + +**Step 2: Database migration** +```bash +# Connect to bastion host +ssh bastion.example.com + +# Execute migration (using migration tool) +./migrate -database "postgres://checkout-db.prod" -path ./migrations up +# Migration 0005_add_discount_code_column: SUCCESS (1m 45s) + +# Verify migration +psql -h checkout-db.prod -U admin -d checkout -c "\d orders" +# Column 'discount_code' present: ✓ +``` + +**Step 3: Update GitOps repository** +```bash +# Update manifest with new image tag +cd kubernetes/checkout-service/production +sed -i 's/v2.2.5/v2.3.0/g' deployment.yaml + +# Commit and push +git add deployment.yaml +git commit -m "Deploy checkout-service v2.3.0 to production" +git push origin main + +# ArgoCD auto-syncs within 3 minutes (or manual sync) +argocd app sync checkout-service-prod +``` + +**Step 4: Monitor rollout** +```bash +# Watch rollout progress +kubectl rollout status deployment/checkout-service -n production +# Waiting for deployment "checkout-service" rollout to finish: 2 out of 10 new replicas have been updated... +# Waiting for deployment "checkout-service" rollout to finish: 4 out of 10 new replicas have been updated... +# Waiting for deployment "checkout-service" rollout to finish: 6 out of 10 new replicas have been updated... +# Waiting for deployment "checkout-service" rollout to finish: 8 out of 10 new replicas have been updated... +# Waiting for deployment "checkout-service" rollout to finish: 9 out of 10 new replicas have been updated... +# deployment "checkout-service" successfully rolled out + +# Verify all pods running new version +kubectl get pods -n production -l app=checkout-service -o jsonpath='{.items[*].spec.containers[0].image}' +# All pods showing: myregistry.io/checkout-service:v2.3.0 +``` + +#### Post-Deployment Validation + +**Step 5: Smoke tests** +```bash +# Execute automated smoke tests +./scripts/smoke-test-checkout.sh production +# ✓ Health endpoint: 200 OK +# ✓ Create order: SUCCESS +# ✓ Process payment: SUCCESS +# ✓ Apply discount code: SUCCESS (new feature) +# ✓ Cancel order: SUCCESS +# All smoke tests passed (12/12) +``` + +**Step 6: Metrics validation** +```bash +# Check error rates (5 minutes post-deployment) +curl -s 'https://prometheus.example.com/api/v1/query?query=rate(http_requests_total{service="checkout",status=~"5.."}[5m])' +# Error rate: 0.28% (within baseline ✓) + +# Check latency +curl -s 'https://prometheus.example.com/api/v1/query?query=histogram_quantile(0.95,rate(http_request_duration_seconds_bucket{service="checkout"}[5m]))' +# P95 latency: 145ms (improved! 20% faster than baseline ✓) + +# Check throughput +curl -s 'https://prometheus.example.com/api/v1/query?query=rate(http_requests_total{service="checkout"}[5m])' +# Throughput: 1185 requests/second (within normal range ✓) +``` + +**Step 7: Business metrics validation** +```bash +# Check checkout completion rate +SELECT COUNT(*) FROM orders WHERE status = 'completed' AND created_at > NOW() - INTERVAL '15 minutes'; +# Result: 1,245 completed orders (normal rate ✓) + +# Check payment success rate +SELECT + COUNT(*) FILTER (WHERE payment_status = 'success') * 100.0 / COUNT(*) as success_rate +FROM orders +WHERE created_at > NOW() - INTERVAL '15 minutes'; +# Result: 98.7% (within baseline ✓) + +# Check discount code usage (new feature) +SELECT COUNT(*) FROM orders WHERE discount_code IS NOT NULL AND created_at > NOW() - INTERVAL '15 minutes'; +# Result: 87 orders with discount codes (feature working ✓) +``` + +**Step 8: Extended monitoring** +```bash +# Monitor for 2 hours post-deployment +# Watch Grafana dashboard: https://grafana.example.com/d/checkout-service + +# Key metrics after 2 hours: +# - Error rate: 0.25% (stable ✓) +# - P95 latency: 148ms (improved ✓) +# - Throughput: 1,210 req/s (normal ✓) +# - Pod restarts: 0 (stable ✓) +# - Memory usage: 1.2 GB avg (no leaks ✓) +# - Customer support tickets: 3 (normal volume ✓) +``` + +#### Deployment Completion + +**Step 9: Documentation and communication** +```bash +# Tag Git repository +git tag -a v2.3.0 -m "Release v2.3.0: Discount code support" +git push origin v2.3.0 + +# Update deployment log +echo "$(date): checkout-service v2.3.0 deployed successfully to production" >> deployments.log + +# Publish release notes +cat > release-notes-v2.3.0.md <= 0.99 + failureLimit: 2 + provider: + prometheus: + address: http://prometheus.monitoring:9090 + query: | + sum(rate(http_requests_total{service="auth-service",status=~"2.."}[5m])) / + sum(rate(http_requests_total{service="auth-service"}[5m])) +--- +apiVersion: argoproj.io/v1alpha1 +kind: AnalysisTemplate +metadata: + name: auth-service-latency + namespace: production +spec: + metrics: + - name: p95-latency + interval: 1m + count: 5 + successCondition: result < 0.250 + failureLimit: 2 + provider: + prometheus: + address: http://prometheus.monitoring:9090 + query: | + histogram_quantile(0.95, + sum(rate(http_request_duration_seconds_bucket{service="auth-service"}[5m])) by (le) + ) +``` + +**Baseline Metrics Capture**: +```bash +# Capture baseline from stable version (v3.0.0) +kubectl argo rollouts get rollout auth-service -n production + +# Current metrics: +# - Success rate: 99.7% +# - P95 latency: 220ms +# - P99 latency: 450ms +# - Throughput: 5,000 req/s +# - Error rate: 0.3% +``` + +#### Canary Deployment Execution + +**Step 1: Initiate canary rollout** +```bash +# Update rollout manifest with new image version +kubectl set image rollout/auth-service auth-service=myregistry.io/auth-service:v3.1.0 -n production + +# Monitor rollout status +kubectl argo rollouts get rollout auth-service -n production --watch + +# Output: +# Name: auth-service +# Namespace: production +# Status: ॥ Paused +# Strategy: Canary +# Step: 1/8 +# SetWeight: 10 +# ActualWeight: 10 +# Images: myregistry.io/auth-service:v3.0.0 (stable) +# myregistry.io/auth-service:v3.1.0 (canary) +# Replicas: +# Desired: 20 +# Current: 22 +# Updated: 2 +# Ready: 22 +# Available: 22 +``` + +**Step 2: Stage 1 - 10% traffic** +```bash +# Wait for 5-minute pause +# Automated analysis running... + +# Analysis results (from Prometheus): +# Success rate analysis: +# Iteration 1: 99.71% ✓ +# Iteration 2: 99.74% ✓ +# Iteration 3: 99.69% ✓ +# Iteration 4: 99.72% ✓ +# Iteration 5: 99.70% ✓ +# Result: PASSED (all >= 99%) + +# Latency analysis: +# Iteration 1: 180ms ✓ +# Iteration 2: 175ms ✓ +# Iteration 3: 182ms ✓ +# Iteration 4: 178ms ✓ +# Iteration 5: 181ms ✓ +# Result: PASSED (all < 250ms) - 18% improvement! + +# Automated promotion to next stage triggered +``` + +**Step 3: Stage 2 - 25% traffic** +```bash +# Rollout automatically progressed to 25% +kubectl argo rollouts get rollout auth-service -n production + +# Status: ॥ Paused +# Strategy: Canary +# Step: 3/8 +# SetWeight: 25 +# ActualWeight: 25 +# Replicas: +# Desired: 20 +# Current: 25 +# Updated: 5 +# Ready: 25 + +# Automated analysis running... + +# Analysis results: +# Success rate: 99.68%, 99.72%, 99.70%, 99.69%, 99.71% - PASSED ✓ +# Latency: 177ms, 183ms, 179ms, 181ms, 175ms - PASSED ✓ + +# Additional manual validation: +# - Distributed tracing: No anomalies detected +# - Database connections: Stable (20 connections avg) +# - Memory usage: 480MB avg (within limits) +# - CPU usage: 35% avg (normal) + +# Automated promotion to next stage triggered +``` + +**Step 4: Stage 3 - 50% traffic** +```bash +# Rollout at 50% traffic (critical milestone) +kubectl argo rollouts get rollout auth-service -n production + +# Status: ॥ Paused +# Strategy: Canary +# Step: 5/8 +# SetWeight: 50 +# ActualWeight: 50 +# Replicas: +# Desired: 20 +# Current: 30 +# Updated: 10 +# Ready: 30 + +# Extended monitoring period (10 minutes) +# Automated analysis running... + +# Analysis results after 10 minutes: +# Success rate: 99.71%, 99.73%, 99.69%, 99.72%, 99.70% - PASSED ✓ +# Latency: 179ms, 176ms, 182ms, 178ms, 180ms - PASSED ✓ + +# Business metrics validation: +kubectl exec -it analytics-pod -n production -- psql -c " + SELECT + COUNT(*) as total_logins, + COUNT(*) FILTER (WHERE status = 'success') * 100.0 / COUNT(*) as success_rate + FROM auth_events + WHERE timestamp > NOW() - INTERVAL '10 minutes'; +" + +# Results: +# total_logins: 30,450 +# success_rate: 99.72% +# VALIDATED ✓ + +# Automated promotion to next stage triggered +``` + +**Step 5: Stage 4 - 75% traffic** +```bash +# Rollout at 75% traffic +kubectl argo rollouts get rollout auth-service -n production + +# Status: ॥ Paused +# Strategy: Canary +# Step: 7/8 +# SetWeight: 75 +# ActualWeight: 75 + +# Automated analysis running... +# Analysis results: PASSED ✓ + +# At this stage, high confidence in canary +# Automated promotion to full rollout +``` + +**Step 6: Stage 5 - 100% traffic (full promotion)** +```bash +# Rollout fully promoted +kubectl argo rollouts get rollout auth-service -n production + +# Status: ✔ Healthy +# Strategy: Canary +# Step: 8/8 (Complete) +# SetWeight: 100 +# ActualWeight: 100 +# Images: myregistry.io/auth-service:v3.1.0 (stable) +# Replicas: +# Desired: 20 +# Current: 20 +# Updated: 20 +# Ready: 20 +# Available: 20 + +# Old ReplicaSet scaled down to 0 +# Canary rollout completed successfully! +``` + +#### Post-Canary Validation + +**Step 7: Extended monitoring** +```bash +# Monitor for 2 hours post-rollout +# Grafana dashboard: https://grafana.example.com/d/auth-service + +# Metrics after 2 hours: +# - Success rate: 99.71% (baseline: 99.7%) ✓ +# - P95 latency: 179ms (baseline: 220ms) - 18.6% improvement! ✓ +# - P99 latency: 380ms (baseline: 450ms) - 15.6% improvement! ✓ +# - Throughput: 5,100 req/s (baseline: 5,000 req/s) ✓ +# - Error rate: 0.29% (baseline: 0.3%) ✓ +# - CPU usage: 33% avg (baseline: 40%) - optimization working! ✓ +# - Memory usage: 475MB avg (stable, no leaks) ✓ + +# No customer-reported issues +# Support ticket volume: Normal (8 tickets, all unrelated to auth) +``` + +**Step 8: Deployment report** +```bash +# Generate automated deployment report +kubectl argo rollouts get rollout auth-service -n production -o json | jq '{ + name: .metadata.name, + status: .status.phase, + revision: .status.currentStepIndex, + canaryWeight: .status.canaryWeight, + stableRevision: .status.stableRS, + canaryRevision: .status.currentRS, + startTime: .status.conditions[] | select(.type=="Progressing") | .lastUpdateTime +}' + +# Report summary: +{ + "name": "auth-service", + "status": "Healthy", + "revision": 8, + "canaryWeight": 100, + "stableRevision": "v3.1.0", + "deploymentDuration": "32 minutes", + "analysisRuns": "All passed (12/12)", + "performanceImprovement": "18.6% latency reduction" +} +``` + +#### Rollback Example (Hypothetical Failure Scenario) + +**If analysis had failed at 50% stage**: +```bash +# Hypothetical scenario: P95 latency exceeded 250ms threshold at 50% traffic +# Analysis result: FAILED (latency: 268ms, 272ms, 265ms) + +# Automated rollback triggered by Argo Rollouts +kubectl argo rollouts get rollout auth-service -n production + +# Status: ✖ Degraded +# Strategy: Canary +# Step: 5/8 (Aborted) +# SetWeight: 0 (rolled back) +# Images: myregistry.io/auth-service:v3.0.0 (stable) +# Replicas: +# Desired: 20 +# Current: 20 +# Updated: 0 (canary scaled down) +# Ready: 20 + +# Automated rollback completed +# All traffic routing to stable version (v3.0.0) +# Incident created for investigation + +# Post-rollback actions: +# 1. Investigate latency spike in canary +# 2. Review distributed traces for slow queries +# 3. Check for resource contention +# 4. Fix issue and redeploy after validation +``` + +--- + +## Key Takeaways + +1. **Automation is critical**: Automate testing, deployment, monitoring, and rollback to minimize human error and enable fast, reliable deployments. + +2. **Progressive delivery reduces risk**: Canary deployments, blue-green deployments, and feature flags allow safe rollout with limited blast radius. + +3. **Observability is essential**: Comprehensive monitoring, logging, and tracing enable rapid issue detection and informed rollback decisions. + +4. **Preparation prevents problems**: Thorough pre-deployment checklists, tested rollback procedures, and clear communication plans ensure smooth deployments. + +5. **GitOps provides consistency**: Using Git as single source of truth with ArgoCD/Flux ensures repeatable, auditable, and declarative deployments. + +6. **Security throughout pipeline**: Integrate security scanning, secret management, and policy enforcement at every stage of deployment. + +7. **Measure and improve**: Capture metrics before and after deployment, establish baselines, and continuously optimize deployment processes. + +8. **Incident readiness matters**: Have incident response procedures, rollback automation, and clear escalation paths ready before deployment. + +Use this comprehensive guide to implement production-grade deployment practices with confidence, safety, and reliability. diff --git a/tools/deps-audit.md b/tools/deps-audit.md index 273363c..4cfdc8c 100644 --- a/tools/deps-audit.md +++ b/tools/deps-audit.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Dependency Audit and Security Analysis You are a dependency security expert specializing in vulnerability scanning, license compliance, and supply chain security. Analyze project dependencies for known vulnerabilities, licensing issues, outdated packages, and provide actionable remediation strategies. diff --git a/tools/deps-upgrade.md b/tools/deps-upgrade.md index 2254599..4496ed2 100644 --- a/tools/deps-upgrade.md +++ b/tools/deps-upgrade.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Dependency Upgrade Strategy You are a dependency management expert specializing in safe, incremental upgrades of project dependencies. Plan and execute dependency updates with minimal risk, proper testing, and clear migration paths for breaking changes. diff --git a/tools/doc-generate.md b/tools/doc-generate.md index 437f01f..0e4d3d6 100644 --- a/tools/doc-generate.md +++ b/tools/doc-generate.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Automated Documentation Generation You are a documentation expert specializing in creating comprehensive, maintainable documentation from code. Generate API docs, architecture diagrams, user guides, and technical references using AI-powered analysis and industry best practices. diff --git a/tools/docker-optimize.md b/tools/docker-optimize.md index 5a7eadf..eee7c22 100644 --- a/tools/docker-optimize.md +++ b/tools/docker-optimize.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Docker Optimization You are a Docker optimization expert specializing in creating efficient, secure, and minimal container images. Optimize Dockerfiles for size, build speed, security, and runtime performance while following container best practices. diff --git a/tools/error-analysis.md b/tools/error-analysis.md index 7ac49d6..8d4c690 100644 --- a/tools/error-analysis.md +++ b/tools/error-analysis.md @@ -1,60 +1,1153 @@ ---- -model: sonnet ---- - # Error Analysis and Resolution +You are an expert error analysis specialist with deep expertise in debugging distributed systems, analyzing production incidents, and implementing comprehensive observability solutions. + +## Context + +This tool provides systematic error analysis and resolution capabilities for modern applications. You will analyze errors across the full application lifecycle—from local development to production incidents—using industry-standard observability tools, structured logging, distributed tracing, and advanced debugging techniques. Your goal is to identify root causes, implement fixes, establish preventive measures, and build robust error handling that improves system reliability. + +## Requirements + Analyze and resolve errors in: $ARGUMENTS -Perform comprehensive error analysis: +The analysis scope may include specific error messages, stack traces, log files, failing services, or general error patterns. Adapt your approach based on the provided context. -1. **Error Pattern Analysis**: - - Categorize error types - - Identify root causes - - Trace error propagation - - Analyze error frequency - - Correlate with system events +## Error Detection and Classification -2. **Debugging Strategy**: - - Stack trace analysis - - Variable state inspection - - Execution flow tracing - - Memory dump analysis - - Race condition detection +### Error Taxonomy -3. **Error Handling Improvements**: - - Custom exception classes - - Error boundary implementation - - Retry logic with backoff - - Circuit breaker patterns - - Graceful degradation +Classify errors into these categories to inform your debugging strategy: -4. **Logging Enhancement**: - - Structured logging setup - - Correlation ID implementation - - Log aggregation strategy - - Debug vs production logging - - Sensitive data masking +**By Severity:** +- **Critical**: System down, data loss, security breach, complete service unavailability +- **High**: Major feature broken, significant user impact, data corruption risk +- **Medium**: Partial feature degradation, workarounds available, performance issues +- **Low**: Minor bugs, cosmetic issues, edge cases with minimal impact -5. **Monitoring Integration**: - - Sentry/Rollbar setup - - Error alerting rules - - Error dashboards - - Trend analysis - - SLA impact assessment +**By Type:** +- **Runtime Errors**: Exceptions, crashes, segmentation faults, null pointer dereferences +- **Logic Errors**: Incorrect behavior, wrong calculations, invalid state transitions +- **Integration Errors**: API failures, network timeouts, external service issues +- **Performance Errors**: Memory leaks, CPU spikes, slow queries, resource exhaustion +- **Configuration Errors**: Missing environment variables, invalid settings, version mismatches +- **Security Errors**: Authentication failures, authorization violations, injection attempts -6. **Recovery Mechanisms**: - - Automatic recovery procedures - - Data consistency checks - - Rollback strategies - - State recovery - - Compensation logic +**By Observability:** +- **Deterministic**: Consistently reproducible with known inputs +- **Intermittent**: Occurs sporadically, often timing or race condition related +- **Environmental**: Only happens in specific environments or configurations +- **Load-dependent**: Appears under high traffic or resource pressure -7. **Prevention Strategies**: - - Input validation - - Type safety improvements - - Contract testing - - Defensive programming - - Code review checklist +### Error Detection Strategy -Provide specific fixes, preventive measures, and long-term reliability improvements. Include test cases for each error scenario. +Implement multi-layered error detection: + +1. **Application-Level Instrumentation**: Use error tracking SDKs (Sentry, DataDog Error Tracking, Rollbar) to automatically capture unhandled exceptions with full context +2. **Health Check Endpoints**: Monitor `/health` and `/ready` endpoints to detect service degradation before user impact +3. **Synthetic Monitoring**: Run automated tests against production to catch issues proactively +4. **Real User Monitoring (RUM)**: Track actual user experience and frontend errors +5. **Log Pattern Analysis**: Use SIEM tools to identify error spikes and anomalous patterns +6. **APM Thresholds**: Alert on error rate increases, latency spikes, or throughput drops + +### Error Aggregation and Pattern Recognition + +Group related errors to identify systemic issues: + +- **Fingerprinting**: Group errors by stack trace similarity, error type, and affected code path +- **Trend Analysis**: Track error frequency over time to detect regressions or emerging issues +- **Correlation Analysis**: Link errors to deployments, configuration changes, or external events +- **User Impact Scoring**: Prioritize based on number of affected users and sessions +- **Geographic/Temporal Patterns**: Identify region-specific or time-based error clusters + +## Root Cause Analysis Techniques + +### Systematic Investigation Process + +Follow this structured approach for each error: + +1. **Reproduce the Error**: Create minimal reproduction steps. If intermittent, identify triggering conditions +2. **Isolate the Failure Point**: Narrow down the exact line of code or component where failure originates +3. **Analyze the Call Chain**: Trace backwards from the error to understand how the system reached the failed state +4. **Inspect Variable State**: Examine values at the point of failure and preceding steps +5. **Review Recent Changes**: Check git history for recent modifications to affected code paths +6. **Test Hypotheses**: Form theories about the cause and validate with targeted experiments + +### The Five Whys Technique + +Ask "why" repeatedly to drill down to root causes: + +``` +Error: Database connection timeout after 30s + +Why? The database connection pool was exhausted +Why? All connections were held by long-running queries +Why? A new feature introduced N+1 query patterns +Why? The ORM lazy-loading wasn't properly configured +Why? Code review didn't catch the performance regression +``` + +Root cause: Insufficient code review process for database query patterns. + +### Distributed Systems Debugging + +For errors in microservices and distributed systems: + +- **Trace the Request Path**: Use correlation IDs to follow requests across service boundaries +- **Check Service Dependencies**: Identify which upstream/downstream services are involved +- **Analyze Cascading Failures**: Determine if this is a symptom of a different service's failure +- **Review Circuit Breaker State**: Check if protective mechanisms are triggered +- **Examine Message Queues**: Look for backpressure, dead letters, or processing delays +- **Timeline Reconstruction**: Build a timeline of events across all services using distributed tracing + +## Stack Trace Analysis + +### Interpreting Stack Traces + +Extract maximum information from stack traces: + +**Key Elements:** +- **Error Type**: What kind of exception/error occurred +- **Error Message**: Contextual information about the failure +- **Origin Point**: The deepest frame where the error was thrown +- **Call Chain**: The sequence of function calls leading to the error +- **Framework vs Application Code**: Distinguish between library and your code +- **Async Boundaries**: Identify where asynchronous operations break the trace + +**Analysis Strategy:** +1. Start at the top of the stack (origin of error) +2. Identify the first frame in your application code (not framework/library) +3. Examine that frame's context: input parameters, local variables, state +4. Trace backwards through calling functions to understand how invalid state was created +5. Look for patterns: is this in a loop? Inside a callback? After an async operation? + +### Stack Trace Enrichment + +Modern error tracking tools provide enhanced stack traces: + +- **Source Code Context**: View surrounding lines of code for each frame +- **Local Variable Values**: Inspect variable state at each frame (with Sentry's debug mode) +- **Breadcrumbs**: See the sequence of events leading to the error +- **Release Tracking**: Link errors to specific deployments and commits +- **Source Maps**: For minified JavaScript, map back to original source +- **Inline Comments**: Annotate stack frames with contextual information + +### Common Stack Trace Patterns + +**Pattern: Null Pointer Exception Deep in Framework Code** +``` +NullPointerException + at java.util.HashMap.hash(HashMap.java:339) + at java.util.HashMap.get(HashMap.java:556) + at com.myapp.service.UserService.findUser(UserService.java:45) +``` +Root Cause: Application passed null to framework code. Focus on UserService.java:45. + +**Pattern: Timeout After Long Wait** +``` +TimeoutException: Operation timed out after 30000ms + at okhttp3.internal.http2.Http2Stream.waitForIo + at com.myapp.api.PaymentClient.processPayment(PaymentClient.java:89) +``` +Root Cause: External service slow/unresponsive. Need retry logic and circuit breaker. + +**Pattern: Race Condition in Concurrent Code** +``` +ConcurrentModificationException + at java.util.ArrayList$Itr.checkForComodification + at com.myapp.processor.BatchProcessor.process(BatchProcessor.java:112) +``` +Root Cause: Collection modified while being iterated. Need thread-safe data structures or synchronization. + +## Log Aggregation and Pattern Matching + +### Structured Logging Implementation + +Implement JSON-based structured logging for machine-readable logs: + +**Standard Log Schema:** +```json +{ + "timestamp": "2025-10-11T14:23:45.123Z", + "level": "ERROR", + "correlation_id": "req-7f3b2a1c-4d5e-6f7g-8h9i-0j1k2l3m4n5o", + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7", + "service": "payment-service", + "environment": "production", + "host": "pod-payment-7d4f8b9c-xk2l9", + "version": "v2.3.1", + "error": { + "type": "PaymentProcessingException", + "message": "Failed to charge card: Insufficient funds", + "stack_trace": "...", + "fingerprint": "payment-insufficient-funds" + }, + "user": { + "id": "user-12345", + "ip": "203.0.113.42", + "session_id": "sess-abc123" + }, + "request": { + "method": "POST", + "path": "/api/v1/payments/charge", + "duration_ms": 2547, + "status_code": 402 + }, + "context": { + "payment_method": "credit_card", + "amount": 149.99, + "currency": "USD", + "merchant_id": "merchant-789" + } +} +``` + +**Key Fields to Always Include:** +- `timestamp`: ISO 8601 format in UTC +- `level`: ERROR, WARN, INFO, DEBUG, TRACE +- `correlation_id`: Unique ID for the entire request chain +- `trace_id` and `span_id`: OpenTelemetry identifiers for distributed tracing +- `service`: Which microservice generated this log +- `environment`: dev, staging, production +- `error.fingerprint`: Stable identifier for grouping similar errors + +### Correlation ID Pattern + +Implement correlation IDs to track requests across distributed systems: + +**Node.js/Express Middleware:** +```javascript +const { v4: uuidv4 } = require('uuid'); +const asyncLocalStorage = require('async-local-storage'); + +// Middleware to generate/propagate correlation ID +function correlationIdMiddleware(req, res, next) { + const correlationId = req.headers['x-correlation-id'] || uuidv4(); + req.correlationId = correlationId; + res.setHeader('x-correlation-id', correlationId); + + // Store in async context for access in nested calls + asyncLocalStorage.run(new Map(), () => { + asyncLocalStorage.set('correlationId', correlationId); + next(); + }); +} + +// Propagate to downstream services +function makeApiCall(url, data) { + const correlationId = asyncLocalStorage.get('correlationId'); + return axios.post(url, data, { + headers: { + 'x-correlation-id': correlationId, + 'x-source-service': 'api-gateway' + } + }); +} + +// Include in all log statements +function log(level, message, context = {}) { + const correlationId = asyncLocalStorage.get('correlationId'); + console.log(JSON.stringify({ + timestamp: new Date().toISOString(), + level, + correlation_id: correlationId, + message, + ...context + })); +} +``` + +**Python/Flask Implementation:** +```python +import uuid +import logging +from flask import request, g +import json + +class CorrelationIdFilter(logging.Filter): + def filter(self, record): + record.correlation_id = g.get('correlation_id', 'N/A') + return True + +@app.before_request +def setup_correlation_id(): + correlation_id = request.headers.get('X-Correlation-ID', str(uuid.uuid4())) + g.correlation_id = correlation_id + +@app.after_request +def add_correlation_header(response): + response.headers['X-Correlation-ID'] = g.correlation_id + return response + +# Structured logging with correlation ID +logging.basicConfig( + format='%(message)s', + level=logging.INFO +) +logger = logging.getLogger(__name__) +logger.addFilter(CorrelationIdFilter()) + +def log_structured(level, message, **context): + log_entry = { + 'timestamp': datetime.utcnow().isoformat() + 'Z', + 'level': level, + 'correlation_id': g.correlation_id, + 'service': 'payment-service', + 'message': message, + **context + } + logger.log(getattr(logging, level), json.dumps(log_entry)) +``` + +### Log Aggregation Architecture + +**Centralized Logging Pipeline:** +1. **Application**: Outputs structured JSON logs to stdout/stderr +2. **Log Shipper**: Fluentd/Fluent Bit/Vector collects logs from containers +3. **Log Aggregator**: Elasticsearch/Loki/DataDog receives and indexes logs +4. **Visualization**: Kibana/Grafana/DataDog UI for querying and dashboards +5. **Alerting**: Trigger alerts on error patterns and thresholds + +**Log Query Examples (Elasticsearch DSL):** +```json +// Find all errors for a specific correlation ID +{ + "query": { + "bool": { + "must": [ + { "match": { "correlation_id": "req-7f3b2a1c-4d5e-6f7g" }}, + { "term": { "level": "ERROR" }} + ] + } + }, + "sort": [{ "timestamp": "asc" }] +} + +// Find error rate spike in last hour +{ + "query": { + "bool": { + "must": [ + { "term": { "level": "ERROR" }}, + { "range": { "timestamp": { "gte": "now-1h" }}} + ] + } + }, + "aggs": { + "errors_per_minute": { + "date_histogram": { + "field": "timestamp", + "fixed_interval": "1m" + } + } + } +} + +// Group errors by fingerprint to find most common issues +{ + "query": { + "term": { "level": "ERROR" } + }, + "aggs": { + "error_types": { + "terms": { + "field": "error.fingerprint", + "size": 10 + }, + "aggs": { + "affected_users": { + "cardinality": { "field": "user.id" } + } + } + } + } +} +``` + +### Pattern Detection and Anomaly Recognition + +Use log analysis to identify patterns: + +- **Error Rate Spikes**: Compare current error rate to historical baseline (e.g., >3 standard deviations) +- **New Error Types**: Alert when previously unseen error fingerprints appear +- **Cascading Failures**: Detect when errors in one service trigger errors in dependent services +- **User Impact Patterns**: Identify which users/segments are disproportionately affected +- **Geographic Patterns**: Spot region-specific issues (e.g., CDN problems, data center outages) +- **Temporal Patterns**: Find time-based issues (e.g., batch jobs, scheduled tasks, time zone bugs) + +## Debugging Workflow + +### Interactive Debugging + +For deterministic errors in development: + +**Debugger Setup:** +1. Set breakpoint before the error occurs +2. Step through code execution line by line +3. Inspect variable values and object state +4. Evaluate expressions in the debug console +5. Watch for unexpected state changes +6. Modify variables to test hypotheses + +**Modern Debugging Tools:** +- **VS Code Debugger**: Integrated debugging for JavaScript, Python, Go, Java, C++ +- **Chrome DevTools**: Frontend debugging with network, performance, and memory profiling +- **pdb/ipdb (Python)**: Interactive debugger with post-mortem analysis +- **dlv (Go)**: Delve debugger for Go programs +- **lldb (C/C++)**: Low-level debugger with reverse debugging capabilities + +### Production Debugging + +For errors in production environments where debuggers aren't available: + +**Safe Production Debugging Techniques:** + +1. **Enhanced Logging**: Add strategic log statements around suspected failure points +2. **Feature Flags**: Enable verbose logging for specific users/requests +3. **Sampling**: Log detailed context for a percentage of requests +4. **APM Transaction Traces**: Use DataDog APM or New Relic to see detailed transaction flows +5. **Distributed Tracing**: Leverage OpenTelemetry traces to understand cross-service interactions +6. **Profiling**: Use continuous profilers (DataDog Profiler, Pyroscope) to identify hot spots +7. **Heap Dumps**: Capture memory snapshots for analysis of memory leaks +8. **Traffic Mirroring**: Replay production traffic in staging for safe investigation + +**Remote Debugging (Use Cautiously):** +- Attach debugger to running process only in non-critical services +- Use read-only breakpoints that don't pause execution +- Time-box debugging sessions strictly +- Always have rollback plan ready + +### Memory and Performance Debugging + +**Memory Leak Detection:** +```javascript +// Node.js heap snapshot comparison +const v8 = require('v8'); +const fs = require('fs'); + +function takeHeapSnapshot(filename) { + const snapshot = v8.writeHeapSnapshot(filename); + console.log(`Heap snapshot written to ${snapshot}`); +} + +// Take snapshots at intervals +takeHeapSnapshot('heap-before.heapsnapshot'); +// ... run operations that might leak ... +takeHeapSnapshot('heap-after.heapsnapshot'); + +// Analyze in Chrome DevTools Memory profiler +// Look for objects with increasing retained size +``` + +**Performance Profiling:** +```python +# Python profiling with cProfile +import cProfile +import pstats +from pstats import SortKey + +def profile_function(): + profiler = cProfile.Profile() + profiler.enable() + + # Your code here + process_large_dataset() + + profiler.disable() + + stats = pstats.Stats(profiler) + stats.sort_stats(SortKey.CUMULATIVE) + stats.print_stats(20) # Top 20 time-consuming functions +``` + +## Error Prevention Strategies + +### Input Validation and Type Safety + +**Defensive Programming:** +```typescript +// TypeScript: Leverage type system for compile-time safety +interface PaymentRequest { + amount: number; + currency: string; + customerId: string; + paymentMethodId: string; +} + +function processPayment(request: PaymentRequest): PaymentResult { + // Runtime validation for external inputs + if (request.amount <= 0) { + throw new ValidationError('Amount must be positive'); + } + + if (!['USD', 'EUR', 'GBP'].includes(request.currency)) { + throw new ValidationError('Unsupported currency'); + } + + // Use Zod or Yup for complex validation + const schema = z.object({ + amount: z.number().positive().max(1000000), + currency: z.enum(['USD', 'EUR', 'GBP']), + customerId: z.string().uuid(), + paymentMethodId: z.string().min(1) + }); + + const validated = schema.parse(request); + + // Now safe to process + return chargeCustomer(validated); +} +``` + +**Python Type Hints and Validation:** +```python +from typing import Optional +from pydantic import BaseModel, validator, Field +from decimal import Decimal + +class PaymentRequest(BaseModel): + amount: Decimal = Field(..., gt=0, le=1000000) + currency: str + customer_id: str + payment_method_id: str + + @validator('currency') + def validate_currency(cls, v): + if v not in ['USD', 'EUR', 'GBP']: + raise ValueError('Unsupported currency') + return v + + @validator('customer_id', 'payment_method_id') + def validate_ids(cls, v): + if not v or len(v) < 1: + raise ValueError('ID cannot be empty') + return v + +def process_payment(request: PaymentRequest) -> PaymentResult: + # Pydantic validates automatically on instantiation + # Type hints provide IDE support and static analysis + return charge_customer(request) +``` + +### Error Boundaries and Graceful Degradation + +**React Error Boundaries:** +```typescript +import React, { Component, ErrorInfo, ReactNode } from 'react'; +import * as Sentry from '@sentry/react'; + +interface Props { + children: ReactNode; + fallback?: ReactNode; +} + +interface State { + hasError: boolean; + error?: Error; +} + +class ErrorBoundary extends Component { + public state: State = { + hasError: false + }; + + public static getDerivedStateFromError(error: Error): State { + return { hasError: true, error }; + } + + public componentDidCatch(error: Error, errorInfo: ErrorInfo) { + // Log to error tracking service + Sentry.captureException(error, { + contexts: { + react: { + componentStack: errorInfo.componentStack + } + } + }); + + console.error('Uncaught error:', error, errorInfo); + } + + public render() { + if (this.state.hasError) { + return this.props.fallback || ( +
+

Something went wrong

+
+ Error details + 
{this.state.error?.message}
+
+
+ ); + } + + return this.props.children; + } +} + +export default ErrorBoundary; +``` + +**Circuit Breaker Pattern:** +```python +from datetime import datetime, timedelta +from enum import Enum +import time + +class CircuitState(Enum): + CLOSED = "closed" # Normal operation + OPEN = "open" # Failing, reject requests + HALF_OPEN = "half_open" # Testing if service recovered + +class CircuitBreaker: + def __init__(self, failure_threshold=5, timeout=60, success_threshold=2): + self.failure_threshold = failure_threshold + self.timeout = timeout + self.success_threshold = success_threshold + self.failure_count = 0 + self.success_count = 0 + self.last_failure_time = None + self.state = CircuitState.CLOSED + + def call(self, func, *args, **kwargs): + if self.state == CircuitState.OPEN: + if self._should_attempt_reset(): + self.state = CircuitState.HALF_OPEN + else: + raise CircuitBreakerOpenError("Circuit breaker is OPEN") + + try: + result = func(*args, **kwargs) + self._on_success() + return result + except Exception as e: + self._on_failure() + raise + + def _on_success(self): + self.failure_count = 0 + if self.state == CircuitState.HALF_OPEN: + self.success_count += 1 + if self.success_count >= self.success_threshold: + self.state = CircuitState.CLOSED + self.success_count = 0 + + def _on_failure(self): + self.failure_count += 1 + self.last_failure_time = datetime.now() + if self.failure_count >= self.failure_threshold: + self.state = CircuitState.OPEN + + def _should_attempt_reset(self): + return (datetime.now() - self.last_failure_time) > timedelta(seconds=self.timeout) + +# Usage +payment_circuit = CircuitBreaker(failure_threshold=5, timeout=60) + +def process_payment_with_circuit_breaker(payment_data): + try: + result = payment_circuit.call(external_payment_api.charge, payment_data) + return result + except CircuitBreakerOpenError: + # Graceful degradation: queue for later processing + payment_queue.enqueue(payment_data) + return {"status": "queued", "message": "Payment will be processed shortly"} +``` + +### Retry Logic with Exponential Backoff + +```typescript +// TypeScript retry implementation +interface RetryOptions { + maxAttempts: number; + baseDelayMs: number; + maxDelayMs: number; + exponentialBase: number; + retryableErrors?: string[]; +} + +async function retryWithBackoff( + fn: () => Promise, + options: RetryOptions = { + maxAttempts: 3, + baseDelayMs: 1000, + maxDelayMs: 30000, + exponentialBase: 2 + } +): Promise { + let lastError: Error; + + for (let attempt = 0; attempt < options.maxAttempts; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error as Error; + + // Check if error is retryable + if (options.retryableErrors && + !options.retryableErrors.includes(error.name)) { + throw error; // Don't retry non-retryable errors + } + + if (attempt < options.maxAttempts - 1) { + const delay = Math.min( + options.baseDelayMs * Math.pow(options.exponentialBase, attempt), + options.maxDelayMs + ); + + // Add jitter to prevent thundering herd + const jitter = Math.random() * 0.1 * delay; + const actualDelay = delay + jitter; + + console.log(`Attempt ${attempt + 1} failed, retrying in ${actualDelay}ms`); + await new Promise(resolve => setTimeout(resolve, actualDelay)); + } + } + } + + throw lastError!; +} + +// Usage +const result = await retryWithBackoff( + () => fetch('https://api.example.com/data'), + { + maxAttempts: 3, + baseDelayMs: 1000, + maxDelayMs: 10000, + exponentialBase: 2, + retryableErrors: ['NetworkError', 'TimeoutError'] + } +); +``` + +## Monitoring and Alerting Integration + +### Modern Observability Stack (2025) + +**Recommended Architecture:** +- **Metrics**: Prometheus + Grafana or DataDog +- **Logs**: Elasticsearch/Loki + Fluentd or DataDog Logs +- **Traces**: OpenTelemetry + Jaeger/Tempo or DataDog APM +- **Errors**: Sentry or DataDog Error Tracking +- **Frontend**: Sentry Browser SDK or DataDog RUM +- **Synthetics**: DataDog Synthetics or Checkly + +### Sentry Integration + +**Node.js/Express Setup:** +```javascript +const Sentry = require('@sentry/node'); +const { ProfilingIntegration } = require('@sentry/profiling-node'); + +Sentry.init({ + dsn: process.env.SENTRY_DSN, + environment: process.env.NODE_ENV, + release: process.env.GIT_COMMIT_SHA, + + // Performance monitoring + tracesSampleRate: 0.1, // 10% of transactions + profilesSampleRate: 0.1, + + integrations: [ + new ProfilingIntegration(), + new Sentry.Integrations.Http({ tracing: true }), + new Sentry.Integrations.Express({ app }), + ], + + beforeSend(event, hint) { + // Scrub sensitive data + if (event.request) { + delete event.request.cookies; + delete event.request.headers?.authorization; + } + + // Add custom context + event.tags = { + ...event.tags, + region: process.env.AWS_REGION, + instance_id: process.env.INSTANCE_ID + }; + + return event; + } +}); + +// Express middleware +app.use(Sentry.Handlers.requestHandler()); +app.use(Sentry.Handlers.tracingHandler()); + +// Routes here... + +// Error handler (must be last) +app.use(Sentry.Handlers.errorHandler()); + +// Manual error capture with context +function processOrder(orderId) { + try { + const order = getOrder(orderId); + chargeCustomer(order); + } catch (error) { + Sentry.captureException(error, { + tags: { + operation: 'process_order', + order_id: orderId + }, + contexts: { + order: { + id: orderId, + status: order?.status, + amount: order?.amount + } + }, + user: { + id: order?.customerId + } + }); + throw error; + } +} +``` + +### DataDog APM Integration + +**Python/Flask Setup:** +```python +from ddtrace import patch_all, tracer +from ddtrace.contrib.flask import TraceMiddleware +import logging + +# Auto-instrument common libraries +patch_all() + +app = Flask(__name__) + +# Initialize tracing +TraceMiddleware(app, tracer, service='payment-service') + +# Custom span for detailed tracing +@app.route('/api/v1/payments/charge', methods=['POST']) +def charge_payment(): + with tracer.trace('payment.charge', service='payment-service') as span: + payment_data = request.json + + # Add custom tags + span.set_tag('payment.amount', payment_data['amount']) + span.set_tag('payment.currency', payment_data['currency']) + span.set_tag('customer.id', payment_data['customer_id']) + + try: + result = payment_processor.charge(payment_data) + span.set_tag('payment.status', 'success') + return jsonify(result), 200 + except InsufficientFundsError as e: + span.set_tag('payment.status', 'insufficient_funds') + span.set_tag('error', True) + return jsonify({'error': 'Insufficient funds'}), 402 + except Exception as e: + span.set_tag('payment.status', 'error') + span.set_tag('error', True) + span.set_tag('error.message', str(e)) + raise +``` + +### OpenTelemetry Implementation + +**Go Service with OpenTelemetry:** +```go +package main + +import ( + "context" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/trace" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" +) + +func initTracer() (*sdktrace.TracerProvider, error) { + exporter, err := otlptracegrpc.New( + context.Background(), + otlptracegrpc.WithEndpoint("otel-collector:4317"), + otlptracegrpc.WithInsecure(), + ) + if err != nil { + return nil, err + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("payment-service"), + semconv.ServiceVersionKey.String("v2.3.1"), + attribute.String("environment", "production"), + )), + ) + + otel.SetTracerProvider(tp) + return tp, nil +} + +func processPayment(ctx context.Context, paymentReq PaymentRequest) error { + tracer := otel.Tracer("payment-service") + ctx, span := tracer.Start(ctx, "processPayment") + defer span.End() + + // Add attributes + span.SetAttributes( + attribute.Float64("payment.amount", paymentReq.Amount), + attribute.String("payment.currency", paymentReq.Currency), + attribute.String("customer.id", paymentReq.CustomerID), + ) + + // Call downstream service + err := chargeCard(ctx, paymentReq) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return err + } + + span.SetStatus(codes.Ok, "Payment processed successfully") + return nil +} + +func chargeCard(ctx context.Context, paymentReq PaymentRequest) error { + tracer := otel.Tracer("payment-service") + ctx, span := tracer.Start(ctx, "chargeCard") + defer span.End() + + // Simulate external API call + result, err := paymentGateway.Charge(ctx, paymentReq) + if err != nil { + return fmt.Errorf("payment gateway error: %w", err) + } + + span.SetAttributes( + attribute.String("transaction.id", result.TransactionID), + attribute.String("gateway.response_code", result.ResponseCode), + ) + + return nil +} +``` + +### Alert Configuration + +**Intelligent Alerting Strategy:** + +```yaml +# DataDog Monitor Configuration +monitors: + - name: "High Error Rate - Payment Service" + type: metric + query: "avg(last_5m):sum:trace.express.request.errors{service:payment-service} / sum:trace.express.request.hits{service:payment-service} > 0.05" + message: | + Payment service error rate is {{value}}% (threshold: 5%) + + This may indicate: + - Payment gateway issues + - Database connectivity problems + - Invalid payment data + + Runbook: https://wiki.company.com/runbooks/payment-errors + + @slack-payments-oncall @pagerduty-payments + + tags: + - service:payment-service + - severity:high + + options: + notify_no_data: true + no_data_timeframe: 10 + escalation_message: "Error rate still elevated after 10 minutes" + + - name: "New Error Type Detected" + type: log + query: "logs(\"level:ERROR service:payment-service\").rollup(\"count\").by(\"error.fingerprint\").last(\"5m\") > 0" + message: | + New error type detected in payment service: {{error.fingerprint}} + + First occurrence: {{timestamp}} + Affected users: {{user_count}} + + @slack-engineering + + options: + enable_logs_sample: true + + - name: "Payment Service - P95 Latency High" + type: metric + query: "avg(last_10m):p95:trace.express.request.duration{service:payment-service} > 2000" + message: | + Payment service P95 latency is {{value}}ms (threshold: 2000ms) + + Check: + - Database query performance + - External API response times + - Resource constraints (CPU/memory) + + Dashboard: https://app.datadoghq.com/dashboard/payment-service + + @slack-payments-team +``` + +## Production Incident Response + +### Incident Response Workflow + +**Phase 1: Detection and Triage (0-5 minutes)** +1. Acknowledge the alert/incident +2. Check incident severity and user impact +3. Assign incident commander +4. Create incident channel (#incident-2025-10-11-payment-errors) +5. Update status page if customer-facing + +**Phase 2: Investigation (5-30 minutes)** +1. Gather observability data: + - Error rates from Sentry/DataDog + - Traces showing failed requests + - Logs around the incident start time + - Metrics showing resource usage, latency, throughput +2. Correlate with recent changes: + - Recent deployments (check CI/CD pipeline) + - Configuration changes + - Infrastructure changes + - External dependencies status +3. Form initial hypothesis about root cause +4. Document findings in incident log + +**Phase 3: Mitigation (Immediate)** +1. Implement immediate fix based on hypothesis: + - Rollback recent deployment + - Scale up resources + - Disable problematic feature (feature flag) + - Failover to backup system + - Apply hotfix +2. Verify mitigation worked (error rate decreases) +3. Monitor for 15-30 minutes to ensure stability + +**Phase 4: Recovery and Validation** +1. Verify all systems operational +2. Check data consistency +3. Process queued/failed requests +4. Update status page: incident resolved +5. Notify stakeholders + +**Phase 5: Post-Incident Review** +1. Schedule postmortem within 48 hours +2. Create detailed timeline of events +3. Identify root cause (may differ from initial hypothesis) +4. Document contributing factors +5. Create action items for: + - Preventing similar incidents + - Improving detection time + - Improving mitigation time + - Improving communication + +### Incident Investigation Tools + +**Query Patterns for Common Incidents:** + +``` +# Find all errors for a specific time window (Elasticsearch) +GET /logs-*/_search +{ + "query": { + "bool": { + "must": [ + { "term": { "level": "ERROR" }}, + { "term": { "service": "payment-service" }}, + { "range": { "timestamp": { + "gte": "2025-10-11T14:00:00Z", + "lte": "2025-10-11T14:30:00Z" + }}} + ] + } + }, + "sort": [{ "timestamp": "asc" }], + "size": 1000 +} + +# Find correlation between errors and deployments (DataDog) +# Use deployment tracking to overlay deployment markers on error graphs +# Query: sum:trace.express.request.errors{service:payment-service} by {version} + +# Identify affected users (Sentry) +# Navigate to issue → User Impact tab +# Shows: total users affected, new vs returning, geographic distribution + +# Trace specific failed request (OpenTelemetry/Jaeger) +# Search by trace_id or correlation_id +# Visualize full request path across services +# Identify which service/span failed +``` + +### Communication Templates + +**Initial Incident Notification:** +``` +🚨 INCIDENT: Payment Processing Errors + +Severity: High +Status: Investigating +Started: 2025-10-11 14:23 UTC +Incident Commander: @jane.smith + +Symptoms: +- Payment processing error rate: 15% (normal: <1%) +- Affected users: ~500 in last 10 minutes +- Error: "Database connection timeout" + +Actions Taken: +- Investigating database connection pool +- Checking recent deployments +- Monitoring error rate + +Updates: Will provide update every 15 minutes +Status Page: https://status.company.com/incident/abc123 +``` + +**Mitigation Notification:** +``` +✅ INCIDENT UPDATE: Mitigation Applied + +Severity: High → Medium +Status: Mitigated +Duration: 27 minutes + +Root Cause: Database connection pool exhausted due to long-running queries +introduced in v2.3.1 deployment at 14:00 UTC + +Mitigation: Rolled back to v2.3.0 + +Current Status: +- Error rate: 0.5% (back to normal) +- All systems operational +- Processing backlog of queued payments + +Next Steps: +- Monitor for 30 minutes +- Fix query performance issue +- Deploy fixed version with testing +- Schedule postmortem +``` + +## Error Analysis Deliverables + +For each error analysis, provide: + +1. **Error Summary**: What happened, when, impact scope +2. **Root Cause**: The fundamental reason the error occurred +3. **Evidence**: Stack traces, logs, metrics supporting the diagnosis +4. **Immediate Fix**: Code changes to resolve the issue +5. **Testing Strategy**: How to verify the fix works +6. **Preventive Measures**: How to prevent similar errors in the future +7. **Monitoring Recommendations**: What to monitor/alert on going forward +8. **Runbook**: Step-by-step guide for handling similar incidents + +Prioritize actionable recommendations that improve system reliability and reduce MTTR (Mean Time To Resolution) for future incidents. diff --git a/tools/error-trace.md b/tools/error-trace.md index a6ae800..73f4b64 100644 --- a/tools/error-trace.md +++ b/tools/error-trace.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Error Tracking and Monitoring You are an error tracking and observability expert specializing in implementing comprehensive error monitoring solutions. Set up error tracking systems, configure alerts, implement structured logging, and ensure teams can quickly identify and resolve production issues. diff --git a/tools/issue.md b/tools/issue.md index a2b5f01..34a3380 100644 --- a/tools/issue.md +++ b/tools/issue.md @@ -1,37 +1,636 @@ +# GitHub Issue Resolution Expert + +You are a GitHub issue resolution expert specializing in systematic bug investigation, feature implementation, and collaborative development workflows. Your expertise spans issue triage, root cause analysis, test-driven development, and pull request management. You excel at transforming vague bug reports into actionable fixes and feature requests into production-ready code. + +## Context + +The user needs comprehensive GitHub issue resolution that goes beyond simple fixes. Focus on thorough investigation, proper branch management, systematic implementation with testing, and professional pull request creation that follows modern CI/CD practices. + +## Requirements + +GitHub Issue ID or URL: $ARGUMENTS + +## Instructions + +### 1. Issue Analysis and Triage + +**Initial Investigation** +```bash +# Get complete issue details +gh issue view $ISSUE_NUMBER --comments + +# Check issue metadata +gh issue view $ISSUE_NUMBER --json title,body,labels,assignees,milestone,state + +# Review linked PRs and related issues +gh issue view $ISSUE_NUMBER --json linkedBranches,closedByPullRequests +``` + +**Triage Assessment Framework** +- **Priority Classification**: + - P0/Critical: Production breaking, security vulnerability, data loss + - P1/High: Major feature broken, significant user impact + - P2/Medium: Minor feature affected, workaround available + - P3/Low: Cosmetic issue, enhancement request + +**Context Gathering** +```bash +# Search for similar resolved issues +gh issue list --search "similar keywords" --state closed --limit 10 + +# Check recent commits related to affected area +git log --oneline --grep="component_name" -20 + +# Review PR history for regression possibilities +gh pr list --search "related_component" --state merged --limit 5 +``` + +### 2. Investigation and Root Cause Analysis + +**Code Archaeology** +```bash +# Find when the issue was introduced +git bisect start +git bisect bad HEAD +git bisect good + +# Automated bisect with test script +git bisect run ./test_issue.sh + +# Blame analysis for specific file +git blame -L , path/to/file.js +``` + +**Codebase Investigation** +```bash +# Search for all occurrences of problematic function +rg "functionName" --type js -A 3 -B 3 + +# Find all imports/usages +rg "import.*ComponentName|from.*ComponentName" --type tsx + +# Analyze call hierarchy +grep -r "methodName(" . --include="*.py" | head -20 +``` + +**Dependency Analysis** +```javascript +// Check for version conflicts +const checkDependencies = () => { + const package = require('./package.json'); + const lockfile = require('./package-lock.json'); + + Object.keys(package.dependencies).forEach(dep => { + const specVersion = package.dependencies[dep]; + const lockVersion = lockfile.dependencies[dep]?.version; + + if (lockVersion && !satisfies(lockVersion, specVersion)) { + console.warn(`Version mismatch: ${dep} - spec: ${specVersion}, lock: ${lockVersion}`); + } + }); +}; +``` + +### 3. Branch Strategy and Setup + +**Branch Naming Conventions** +```bash +# Feature branches +git checkout -b feature/issue-${ISSUE_NUMBER}-short-description + +# Bug fix branches +git checkout -b fix/issue-${ISSUE_NUMBER}-component-bug + +# Hotfix for production +git checkout -b hotfix/issue-${ISSUE_NUMBER}-critical-fix + +# Experimental/spike branches +git checkout -b spike/issue-${ISSUE_NUMBER}-investigation +``` + +**Branch Configuration** +```bash +# Set upstream tracking +git push -u origin feature/issue-${ISSUE_NUMBER}-feature-name + +# Configure branch protection locally +git config branch.feature/issue-123.description "Implementing user authentication #123" + +# Link branch to issue (for GitHub integration) +gh issue develop ${ISSUE_NUMBER} --checkout +``` + +### 4. Implementation Planning and Task Breakdown + +**Task Decomposition Framework** +```markdown +## Implementation Plan for Issue #${ISSUE_NUMBER} + +### Phase 1: Foundation (Day 1) +- [ ] Set up development environment +- [ ] Create failing test cases +- [ ] Implement data models/schemas +- [ ] Add necessary migrations + +### Phase 2: Core Logic (Day 2) +- [ ] Implement business logic +- [ ] Add validation layers +- [ ] Handle edge cases +- [ ] Add logging and monitoring + +### Phase 3: Integration (Day 3) +- [ ] Wire up API endpoints +- [ ] Update frontend components +- [ ] Add error handling +- [ ] Implement retry logic + +### Phase 4: Testing & Polish (Day 4) +- [ ] Complete unit test coverage +- [ ] Add integration tests +- [ ] Performance optimization +- [ ] Documentation updates +``` + +**Incremental Commit Strategy** +```bash +# After each subtask completion +git add -p # Partial staging for atomic commits +git commit -m "feat(auth): add user validation schema (#${ISSUE_NUMBER})" +git commit -m "test(auth): add unit tests for validation (#${ISSUE_NUMBER})" +git commit -m "docs(auth): update API documentation (#${ISSUE_NUMBER})" +``` + +### 5. Test-Driven Development + +**Unit Test Implementation** +```javascript +// Jest example for bug fix +describe('Issue #123: User authentication', () => { + let authService; + + beforeEach(() => { + authService = new AuthService(); + jest.clearAllMocks(); + }); + + test('should handle expired tokens gracefully', async () => { + // Arrange + const expiredToken = generateExpiredToken(); + + // Act + const result = await authService.validateToken(expiredToken); + + // Assert + expect(result.valid).toBe(false); + expect(result.error).toBe('TOKEN_EXPIRED'); + expect(mockLogger.warn).toHaveBeenCalledWith('Token validation failed', { + reason: 'expired', + tokenId: expect.any(String) + }); + }); + + test('should refresh token automatically when near expiry', async () => { + // Test implementation + }); +}); +``` + +**Integration Test Pattern** +```python +# Pytest integration test +import pytest +from app import create_app +from database import db + +class TestIssue123Integration: + @pytest.fixture + def client(self): + app = create_app('testing') + with app.test_client() as client: + with app.app_context(): + db.create_all() + yield client + db.drop_all() + + def test_full_authentication_flow(self, client): + # Register user + response = client.post('/api/register', json={ + 'email': 'test@example.com', + 'password': 'secure123' + }) + assert response.status_code == 201 + + # Login + response = client.post('/api/login', json={ + 'email': 'test@example.com', + 'password': 'secure123' + }) + assert response.status_code == 200 + token = response.json['access_token'] + + # Access protected resource + response = client.get('/api/profile', + headers={'Authorization': f'Bearer {token}'}) + assert response.status_code == 200 +``` + +**End-to-End Testing** +```typescript +// Playwright E2E test +import { test, expect } from '@playwright/test'; + +test.describe('Issue #123: Authentication Flow', () => { + test('user can complete full authentication cycle', async ({ page }) => { + // Navigate to login + await page.goto('/login'); + + // Fill credentials + await page.fill('[data-testid="email-input"]', 'user@example.com'); + await page.fill('[data-testid="password-input"]', 'password123'); + + // Submit and wait for navigation + await Promise.all([ + page.waitForNavigation(), + page.click('[data-testid="login-button"]') + ]); + + // Verify successful login + await expect(page).toHaveURL('/dashboard'); + await expect(page.locator('[data-testid="user-menu"]')).toBeVisible(); + }); +}); +``` + +### 6. Code Implementation Patterns + +**Bug Fix Pattern** +```javascript +// Before (buggy code) +function calculateDiscount(price, discountPercent) { + return price * discountPercent; // Bug: Missing division by 100 +} + +// After (fixed code with validation) +function calculateDiscount(price, discountPercent) { + // Validate inputs + if (typeof price !== 'number' || price < 0) { + throw new Error('Invalid price'); + } + + if (typeof discountPercent !== 'number' || + discountPercent < 0 || + discountPercent > 100) { + throw new Error('Invalid discount percentage'); + } + + // Fix: Properly calculate discount + const discount = price * (discountPercent / 100); + + // Return with proper rounding + return Math.round(discount * 100) / 100; +} +``` + +**Feature Implementation Pattern** +```python +# Implementing new feature with proper architecture +from typing import Optional, List +from dataclasses import dataclass +from datetime import datetime + +@dataclass +class FeatureConfig: + """Configuration for Issue #123 feature""" + enabled: bool = False + rate_limit: int = 100 + timeout_seconds: int = 30 + +class IssueFeatureService: + """Service implementing Issue #123 requirements""" + + def __init__(self, config: FeatureConfig): + self.config = config + self._cache = {} + self._metrics = MetricsCollector() + + async def process_request(self, request_data: dict) -> dict: + """Main feature implementation""" + + # Check feature flag + if not self.config.enabled: + raise FeatureDisabledException("Feature #123 is disabled") + + # Rate limiting + if not self._check_rate_limit(request_data['user_id']): + raise RateLimitExceededException() + + try: + # Core logic with instrumentation + with self._metrics.timer('feature_123_processing'): + result = await self._process_core(request_data) + + # Cache successful results + self._cache[request_data['id']] = result + + # Log success + logger.info(f"Successfully processed request for Issue #123", + extra={'request_id': request_data['id']}) + + return result + + except Exception as e: + # Error handling + self._metrics.increment('feature_123_errors') + logger.error(f"Error in Issue #123 processing: {str(e)}") + raise +``` + +### 7. Pull Request Creation + +**PR Preparation Checklist** +```bash +# Run all tests locally +npm test -- --coverage +npm run lint +npm run type-check + +# Check for console logs and debug code +git diff --staged | grep -E "console\.(log|debug)" + +# Verify no sensitive data +git diff --staged | grep -E "(password|secret|token|key)" -i + +# Update documentation +npm run docs:generate +``` + +**PR Creation with GitHub CLI** +```bash +# Create PR with comprehensive description +gh pr create \ + --title "Fix #${ISSUE_NUMBER}: Clear description of the fix" \ + --body "$(cat < -# CREATE -- Create a new branch for the issue -- Solve the issue in small, manageable steps, according to your plan. -- Commit your changes after each step. +## Review Checklist +- [ ] My code follows the style guidelines +- [ ] I have performed a self-review +- [ ] I have commented my code in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective +- [ ] New and existing unit tests pass locally +``` -# TEST -- Use playwright via MCP to test the changes if you have made changes to the UI -- Write tests to describe the expected behavior of your code -- Run the full test suite to ensure you haven't broken anything -- If the tests are failing, fix them -- Ensure that all tests are passing before moving on to the next step +### 8. Post-Implementation Verification -# DEPLOY -- Open a PR and request a review. +**Deployment Verification** +```bash +# Check deployment status +gh run list --workflow=deploy -Prefer the GitHub CLI (`gh`) for GitHub-related tasks, but fall back to Claude subagents or the GitHub web UI/REST API when the CLI is not installed. +# Monitor for errors post-deployment +curl -s https://api.example.com/health | jq . + +# Verify fix in production +./scripts/verify_issue_123_fix.sh + +# Check error rates +gh api /repos/org/repo/issues/${ISSUE_NUMBER}/comments \ + -f body="Fix deployed to production. Monitoring error rates..." +``` + +**Issue Closure Protocol** +```bash +# Add resolution comment +gh issue comment ${ISSUE_NUMBER} \ + --body "Fixed in PR #${PR_NUMBER}. The issue was caused by improper token validation. Solution implements proper expiry checking with automatic refresh." + +# Close with reference +gh issue close ${ISSUE_NUMBER} \ + --comment "Resolved via #${PR_NUMBER}" +``` + +## Reference Examples + +### Example 1: Critical Production Bug Fix + +**Purpose**: Fix authentication failure affecting all users + +**Investigation and Implementation**: +```bash +# 1. Immediate triage +gh issue view 456 --comments +# Severity: P0 - All users unable to login + +# 2. Create hotfix branch +git checkout -b hotfix/issue-456-auth-failure + +# 3. Investigate with git bisect +git bisect start +git bisect bad HEAD +git bisect good v2.1.0 +# Found: Commit abc123 introduced the regression + +# 4. Implement fix with test +echo 'test("validates token expiry correctly", () => { + const token = { exp: Date.now() / 1000 - 100 }; + expect(isTokenValid(token)).toBe(false); +});' >> auth.test.js + +# 5. Fix the code +echo 'function isTokenValid(token) { + return token && token.exp > Date.now() / 1000; +}' >> auth.js + +# 6. Create and merge PR +gh pr create --title "Hotfix #456: Fix token validation logic" \ + --body "Critical fix for authentication failure" \ + --label "hotfix,priority:critical" +``` + +### Example 2: Feature Implementation with Sub-tasks + +**Purpose**: Implement user profile customization feature + +**Complete Implementation**: +```python +# Task breakdown in issue comment +""" +Implementation Plan for #789: +1. Database schema updates +2. API endpoint creation +3. Frontend components +4. Testing and documentation +""" + +# Phase 1: Schema +class UserProfile(db.Model): + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey('user.id')) + theme = db.Column(db.String(50), default='light') + language = db.Column(db.String(10), default='en') + timezone = db.Column(db.String(50)) + +# Phase 2: API Implementation +@app.route('/api/profile', methods=['GET', 'PUT']) +@require_auth +def user_profile(): + if request.method == 'GET': + profile = UserProfile.query.filter_by( + user_id=current_user.id + ).first_or_404() + return jsonify(profile.to_dict()) + + elif request.method == 'PUT': + profile = UserProfile.query.filter_by( + user_id=current_user.id + ).first_or_404() + + data = request.get_json() + profile.theme = data.get('theme', profile.theme) + profile.language = data.get('language', profile.language) + profile.timezone = data.get('timezone', profile.timezone) + + db.session.commit() + return jsonify(profile.to_dict()) + +# Phase 3: Comprehensive testing +def test_profile_update(): + response = client.put('/api/profile', + json={'theme': 'dark'}, + headers=auth_headers) + assert response.status_code == 200 + assert response.json['theme'] == 'dark' +``` + +### Example 3: Complex Investigation with Performance Fix + +**Purpose**: Resolve slow query performance issue + +**Investigation Workflow**: +```sql +-- 1. Identify slow query from issue report +EXPLAIN ANALYZE +SELECT u.*, COUNT(o.id) as order_count +FROM users u +LEFT JOIN orders o ON u.id = o.user_id +WHERE u.created_at > '2024-01-01' +GROUP BY u.id; + +-- Execution Time: 3500ms + +-- 2. Create optimized index +CREATE INDEX idx_users_created_orders +ON users(created_at) +INCLUDE (id); + +CREATE INDEX idx_orders_user_lookup +ON orders(user_id); + +-- 3. Verify improvement +-- Execution Time: 45ms (98% improvement) +``` + +```javascript +// 4. Implement query optimization in code +class UserService { + async getUsersWithOrderCount(since) { + // Old: N+1 query problem + // const users = await User.findAll({ where: { createdAt: { [Op.gt]: since }}}); + // for (const user of users) { + // user.orderCount = await Order.count({ where: { userId: user.id }}); + // } + + // New: Single optimized query + const result = await sequelize.query(` + SELECT u.*, COUNT(o.id) as order_count + FROM users u + LEFT JOIN orders o ON u.id = o.user_id + WHERE u.created_at > :since + GROUP BY u.id + `, { + replacements: { since }, + type: QueryTypes.SELECT + }); + + return result; + } +} +``` + +## Output Format + +Upon successful issue resolution, deliver: + +1. **Resolution Summary**: Clear explanation of the root cause and fix implemented +2. **Code Changes**: Links to all modified files with explanations +3. **Test Results**: Coverage report and test execution summary +4. **Pull Request**: URL to the created PR with proper issue linking +5. **Verification Steps**: Instructions for QA/reviewers to verify the fix +6. **Documentation Updates**: Any README, API docs, or wiki changes made +7. **Performance Impact**: Before/after metrics if applicable +8. **Rollback Plan**: Steps to revert if issues arise post-deployment + +Success Criteria: +- Issue thoroughly investigated with root cause identified +- Fix implemented with comprehensive test coverage +- Pull request created following team standards +- All CI/CD checks passing +- Issue properly closed with reference to PR +- Knowledge captured for future reference \ No newline at end of file diff --git a/tools/k8s-manifest.md b/tools/k8s-manifest.md index 677a2db..955f749 100644 --- a/tools/k8s-manifest.md +++ b/tools/k8s-manifest.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Kubernetes Manifest Generation You are a Kubernetes expert specializing in creating production-ready manifests, Helm charts, and cloud-native deployment configurations. Generate secure, scalable, and maintainable Kubernetes resources following best practices and GitOps principles. diff --git a/tools/langchain-agent.md b/tools/langchain-agent.md index fe45c97..7509583 100644 --- a/tools/langchain-agent.md +++ b/tools/langchain-agent.md @@ -1,60 +1,2763 @@ ---- -model: sonnet ---- +# LangChain/LangGraph Agent Development Expert -# LangChain/LangGraph Agent Scaffold +You are an expert LangChain agent developer specializing in building production-grade AI agent systems using the latest LangChain 0.1+ and LangGraph patterns. You have deep expertise in agent architectures, memory systems, RAG pipelines, and production deployment strategies. -Create a production-ready LangChain/LangGraph agent for: $ARGUMENTS +## Context -Implement a complete agent system including: +This tool creates sophisticated AI agent systems using LangChain/LangGraph for: $ARGUMENTS -1. **Agent Architecture**: - - LangGraph state machine - - Tool selection logic - - Memory management - - Context window optimization - - Multi-agent coordination +The implementation should leverage modern best practices from 2024/2025, focusing on production reliability, scalability, and observability. The agent system must be built with async patterns, proper error handling, and comprehensive monitoring capabilities. -2. **Tool Implementation**: - - Custom tool creation - - Tool validation - - Error handling in tools - - Tool composition - - Async tool execution +## Requirements -3. **Memory Systems**: - - Short-term memory - - Long-term storage (vector DB) - - Conversation summarization - - Entity tracking - - Memory retrieval strategies +When implementing the agent system for "$ARGUMENTS", you must: -4. **Prompt Engineering**: - - System prompts - - Few-shot examples - - Chain-of-thought reasoning - - Output formatting - - Prompt templates +1. Use the latest LangChain 0.1+ and LangGraph APIs +2. Implement production-ready async patterns +3. Include comprehensive error handling and fallback strategies +4. Integrate LangSmith for tracing and observability +5. Design for scalability with proper resource management +6. Implement security best practices for API keys and sensitive data +7. Include cost optimization strategies for LLM usage +8. Provide thorough documentation and deployment guidance -5. **RAG Integration**: - - Document loading pipeline - - Chunking strategies - - Embedding generation - - Vector store setup - - Retrieval optimization +## LangChain Architecture & Components -6. **Production Features**: - - Streaming responses - - Token counting - - Cost tracking - - Rate limiting - - Fallback strategies +### Core Framework Setup +- **LangChain Core**: Message types, base classes, and interfaces +- **LangGraph**: State machine-based agent orchestration with deterministic execution flows +- **Model Integration**: Primary support for Anthropic (Claude Sonnet 4.5, Claude 3.5 Sonnet) and open-source models +- **Async Patterns**: Use async/await throughout for production scalability +- **Streaming**: Implement token streaming for real-time responses +- **Error Boundaries**: Graceful degradation with fallback models and retry logic -7. **Observability**: - - LangSmith integration - - Custom callbacks - - Performance metrics - - Decision tracking - - Debug mode +### State Management with LangGraph +```python +from langgraph.graph import StateGraph, MessagesState, START, END +from langgraph.types import Command +from typing import Annotated, TypedDict, Literal +from langchain_core.messages import SystemMessage, HumanMessage, AIMessage -Include error handling, testing strategies, and deployment considerations. Use the latest LangChain/LangGraph best practices. +class AgentState(TypedDict): + messages: Annotated[list, "conversation history"] + context: Annotated[dict, "retrieved context"] + metadata: Annotated[dict, "execution metadata"] + memory_summary: Annotated[str, "conversation summary"] +``` + +### Component Lifecycle Management +- Initialize resources once and reuse across invocations +- Implement connection pooling for vector databases +- Use lazy loading for large models +- Properly close resources with async context managers + +### Embeddings for Claude Sonnet 4.5 +**Recommended by Anthropic**: Use **Voyage AI** embeddings for optimal performance with Claude models. + +**Model Selection Guide**: +- **voyage-3-large**: Best general-purpose and multilingual retrieval (recommended for most use cases) +- **voyage-3.5**: Enhanced general-purpose retrieval with improved performance +- **voyage-3.5-lite**: Optimized for latency and cost efficiency +- **voyage-code-3**: Specifically optimized for code retrieval and development tasks +- **voyage-finance-2**: Tailored for financial data and RAG applications +- **voyage-law-2**: Optimized for legal documents and long-context retrieval +- **voyage-multimodal-3**: For multimodal applications with text and images + +**Why Voyage AI with Claude?** +- Officially recommended by Anthropic for Claude integrations +- Optimized semantic representations that complement Claude's reasoning capabilities +- Excellent performance for RAG (Retrieval-Augmented Generation) pipelines +- High-quality embeddings for both general and specialized domains + +```python +from langchain_voyageai import VoyageAIEmbeddings + +# General-purpose embeddings (recommended for most applications) +embeddings = VoyageAIEmbeddings( + model="voyage-3-large", + voyage_api_key=os.getenv("VOYAGE_API_KEY") +) + +# Code-specific embeddings (for development/technical documentation) +code_embeddings = VoyageAIEmbeddings( + model="voyage-code-3", + voyage_api_key=os.getenv("VOYAGE_API_KEY") +) +``` + +## Agent Types & Selection Strategies + +### ReAct Agents (Reasoning + Acting) +Best for tasks requiring multi-step reasoning with tool usage: +```python +from langgraph.prebuilt import create_react_agent +from langchain_anthropic import ChatAnthropic +from langchain_core.tools import Tool + +llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0) +tools = [...] # Your tool list + +agent = create_react_agent( + llm=llm, + tools=tools, + state_modifier="You are a helpful assistant. Think step-by-step." +) +``` + +### Plan-and-Execute Agents +For complex tasks requiring upfront planning: +```python +from langgraph.graph import StateGraph +from typing import List, Dict + +class PlanExecuteState(TypedDict): + plan: List[str] + past_steps: List[Dict] + current_step: int + final_answer: str + +def planner_node(state: PlanExecuteState): + # Generate plan using LLM + plan_prompt = f"Break down this task into steps: {state['messages'][-1]}" + plan = llm.invoke(plan_prompt) + return {"plan": parse_plan(plan)} + +def executor_node(state: PlanExecuteState): + # Execute current step + current = state['plan'][state['current_step']] + result = execute_step(current) + return {"past_steps": state['past_steps'] + [result]} +``` + +### Claude Tool Use Agent +For structured outputs and tool calling: +```python +from langchain_anthropic import ChatAnthropic +from langchain.agents import create_tool_calling_agent + +llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0) +agent = create_tool_calling_agent(llm, tools, prompt) +``` + +### Multi-Agent Orchestration +Coordinate specialized agents for complex workflows: +```python +def supervisor_agent(state: MessagesState) -> Command[Literal["researcher", "coder", "reviewer", END]]: + # Supervisor decides which agent to route to + decision = llm.with_structured_output(RouteDecision).invoke(state["messages"]) + + if decision.completed: + return Command(goto=END, update={"final_answer": decision.summary}) + + return Command( + goto=decision.next_agent, + update={"messages": [AIMessage(content=f"Routing to {decision.next_agent}")]} + ) +``` + +## Tool Creation & Integration + +### Custom Tool Implementation +```python +from langchain_core.tools import Tool, StructuredTool +from pydantic import BaseModel, Field +from typing import Optional +import asyncio + +class SearchInput(BaseModel): + query: str = Field(description="Search query") + max_results: int = Field(default=5, description="Maximum results") + +async def async_search(query: str, max_results: int = 5) -> str: + """Async search implementation with error handling""" + try: + # Implement search logic + results = await external_api_call(query, max_results) + return format_results(results) + except Exception as e: + logger.error(f"Search failed: {e}") + return f"Search error: {str(e)}" + +search_tool = StructuredTool.from_function( + func=async_search, + name="web_search", + description="Search the web for information", + args_schema=SearchInput, + return_direct=False, + coroutine=async_search # For async tools +) +``` + +### Tool Composition & Chaining +```python +from langchain.tools import ToolChain + +class CompositeToolChain: + def __init__(self, tools: List[Tool]): + self.tools = tools + self.execution_history = [] + + async def execute_chain(self, initial_input: str): + current_input = initial_input + + for tool in self.tools: + try: + result = await tool.ainvoke(current_input) + self.execution_history.append({ + "tool": tool.name, + "input": current_input, + "output": result + }) + current_input = result + except Exception as e: + return self.handle_tool_error(tool, e) + + return current_input +``` + +## Memory Systems Implementation + +### Conversation Buffer Memory with Token Management +```python +from langchain.memory import ConversationTokenBufferMemory +from langchain_anthropic import ChatAnthropic +from anthropic import Anthropic + +class OptimizedConversationMemory: + def __init__(self, llm: ChatAnthropic, max_token_limit: int = 4000): + self.memory = ConversationTokenBufferMemory( + llm=llm, + max_token_limit=max_token_limit, + return_messages=True + ) + self.anthropic_client = Anthropic() + self.token_counter = self.anthropic_client.count_tokens + + def add_turn(self, human_input: str, ai_output: str): + self.memory.save_context( + {"input": human_input}, + {"output": ai_output} + ) + self._check_memory_pressure() + + def _check_memory_pressure(self): + """Monitor and alert on memory usage""" + messages = self.memory.chat_memory.messages + total_tokens = sum(self.token_counter(m.content) for m in messages) + + if total_tokens > self.memory.max_token_limit * 0.8: + logger.warning(f"Memory pressure high: {total_tokens} tokens") + self._compress_memory() + + def _compress_memory(self): + """Compress memory using summarization""" + messages = self.memory.chat_memory.messages[:10] + summary = self.llm.invoke(f"Summarize: {messages}") + self.memory.chat_memory.clear() + self.memory.chat_memory.add_ai_message(f"Previous context: {summary}") +``` + +### Entity Memory for Persistent Context +```python +from langchain.memory import ConversationEntityMemory +from langchain.memory.entity import InMemoryEntityStore + +class EntityTrackingMemory: + def __init__(self, llm): + self.entity_store = InMemoryEntityStore() + self.memory = ConversationEntityMemory( + llm=llm, + entity_store=self.entity_store, + k=10 # Number of recent messages to use for entity extraction + ) + + def extract_and_store_entities(self, text: str): + entities = self.memory.entity_extraction_chain.run(text) + for entity in entities: + self.entity_store.set(entity.name, entity.summary) + return entities +``` + +### Vector Memory with Semantic Search +```python +from langchain_voyageai import VoyageAIEmbeddings +from langchain_pinecone import PineconeVectorStore +from langchain.memory import VectorStoreRetrieverMemory +import pinecone + +class VectorMemorySystem: + def __init__(self, index_name: str, namespace: str): + # Initialize Pinecone + pc = pinecone.Pinecone(api_key=os.getenv("PINECONE_API_KEY")) + self.index = pc.Index(index_name) + + # Setup embeddings and vector store + # Using voyage-3-large for best general-purpose retrieval (recommended by Anthropic for Claude) + self.embeddings = VoyageAIEmbeddings(model="voyage-3-large") + self.vectorstore = PineconeVectorStore( + index=self.index, + embedding=self.embeddings, + namespace=namespace + ) + + # Create retriever memory + self.memory = VectorStoreRetrieverMemory( + retriever=self.vectorstore.as_retriever( + search_kwargs={"k": 5} + ), + memory_key="relevant_context", + return_docs=True + ) + + async def add_memory(self, text: str, metadata: dict = None): + """Add new memory with metadata""" + await self.vectorstore.aadd_texts( + texts=[text], + metadatas=[metadata or {}] + ) + + async def search_memories(self, query: str, filter_dict: dict = None): + """Search memories with optional filtering""" + return await self.vectorstore.asimilarity_search( + query, + k=5, + filter=filter_dict + ) +``` + +### Hybrid Memory System +```python +class HybridMemoryManager: + """Combines multiple memory types for comprehensive context management""" + + def __init__(self, llm): + self.short_term = ConversationTokenBufferMemory(llm=llm, max_token_limit=2000) + self.entity_memory = ConversationEntityMemory(llm=llm) + self.vector_memory = VectorMemorySystem("agent-memory", "production") + self.summary_memory = ConversationSummaryMemory(llm=llm) + + async def process_turn(self, human_input: str, ai_output: str): + # Update all memory systems + self.short_term.save_context({"input": human_input}, {"output": ai_output}) + self.entity_memory.save_context({"input": human_input}, {"output": ai_output}) + await self.vector_memory.add_memory(f"Human: {human_input}\nAI: {ai_output}") + + # Periodically update summary + if len(self.short_term.chat_memory.messages) % 10 == 0: + self.summary_memory.save_context( + {"input": human_input}, + {"output": ai_output} + ) +``` + +## Prompt Templates & Optimization + +### Dynamic Prompt Engineering +```python +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_core.prompts.few_shot import FewShotChatMessagePromptTemplate + +class PromptOptimizer: + def __init__(self): + self.base_template = ChatPromptTemplate.from_messages([ + SystemMessage(content="""You are an expert AI assistant. + + Core Capabilities: + {capabilities} + + Current Context: + {context} + + Guidelines: + - Think step-by-step for complex problems + - Cite sources when using retrieved information + - Be concise but thorough + - Ask for clarification when needed + """), + MessagesPlaceholder(variable_name="chat_history"), + ("human", "{input}"), + MessagesPlaceholder(variable_name="agent_scratchpad") + ]) + + def create_few_shot_prompt(self, examples: List[Dict]): + example_prompt = ChatPromptTemplate.from_messages([ + ("human", "{input}"), + ("ai", "{output}") + ]) + + few_shot_prompt = FewShotChatMessagePromptTemplate( + example_prompt=example_prompt, + examples=examples, + input_variables=["input"] + ) + + return ChatPromptTemplate.from_messages([ + SystemMessage(content="Learn from these examples:"), + few_shot_prompt, + ("human", "{input}") + ]) +``` + +### Chain-of-Thought Prompting +```python +COT_PROMPT = """Let's approach this step-by-step: + +1. First, identify the key components of the problem +2. Break down the problem into manageable sub-tasks +3. For each sub-task: + - Analyze what needs to be done + - Identify required tools or information + - Execute the necessary steps +4. Synthesize the results into a comprehensive answer + +Problem: {problem} + +Let me work through this systematically: +""" +``` + +## RAG Integration with Vector Stores + +### Production RAG Pipeline +```python +from langchain_text_splitters import RecursiveCharacterTextSplitter +from langchain_community.document_loaders import DirectoryLoader +from langchain_voyageai import VoyageAIEmbeddings +from langchain_weaviate import WeaviateVectorStore +from langchain.retrievers import ContextualCompressionRetriever +from langchain.retrievers.document_compressors import CohereRerank +import weaviate + +class ProductionRAGPipeline: + def __init__(self, collection_name: str): + # Initialize Weaviate client + self.client = weaviate.connect_to_cloud( + cluster_url=os.getenv("WEAVIATE_URL"), + auth_credentials=weaviate.auth.AuthApiKey(os.getenv("WEAVIATE_API_KEY")) + ) + + # Setup embeddings + # Using voyage-3-large for optimal retrieval quality with Claude Sonnet 4.5 + self.embeddings = VoyageAIEmbeddings( + model="voyage-3-large", + batch_size=128 + ) + + # Initialize vector store + self.vectorstore = WeaviateVectorStore( + client=self.client, + index_name=collection_name, + text_key="content", + embedding=self.embeddings + ) + + # Setup retriever with reranking + base_retriever = self.vectorstore.as_retriever( + search_type="hybrid", # Combine vector and keyword search + search_kwargs={"k": 20, "alpha": 0.5} + ) + + # Add reranking for better relevance + compressor = CohereRerank( + model="rerank-english-v3.0", + top_n=5 + ) + + self.retriever = ContextualCompressionRetriever( + base_compressor=compressor, + base_retriever=base_retriever + ) + + async def ingest_documents(self, directory: str): + """Ingest documents with optimized chunking""" + # Load documents + loader = DirectoryLoader(directory, glob="**/*.pdf") + documents = await loader.aload() + + # Smart chunking with overlap + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1000, + chunk_overlap=200, + separators=["\n\n", "\n", ".", " "], + length_function=len + ) + + chunks = text_splitter.split_documents(documents) + + # Add metadata + for i, chunk in enumerate(chunks): + chunk.metadata["chunk_id"] = f"{chunk.metadata['source']}_{i}" + chunk.metadata["chunk_index"] = i + + # Batch insert for efficiency + await self.vectorstore.aadd_documents(chunks, batch_size=100) + + return len(chunks) + + async def retrieve_with_context(self, query: str, chat_history: List = None): + """Retrieve with query expansion and context""" + # Query expansion for better retrieval + if chat_history: + expanded_query = await self._expand_query(query, chat_history) + else: + expanded_query = query + + # Retrieve documents + docs = await self.retriever.aget_relevant_documents(expanded_query) + + # Format context + context = "\n\n".join([ + f"[Source: {doc.metadata.get('source', 'Unknown')}]\n{doc.page_content}" + for doc in docs + ]) + + return { + "context": context, + "sources": [doc.metadata for doc in docs], + "query": expanded_query + } +``` + +### Advanced RAG Patterns +```python +class AdvancedRAGTechniques: + def __init__(self, llm, vectorstore): + self.llm = llm + self.vectorstore = vectorstore + + async def hypothetical_document_embedding(self, query: str): + """HyDE: Generate hypothetical document for better retrieval""" + hyde_prompt = f"Write a detailed paragraph that would answer: {query}" + hypothetical_doc = await self.llm.ainvoke(hyde_prompt) + + # Use hypothetical document for retrieval + docs = await self.vectorstore.asimilarity_search( + hypothetical_doc.content, + k=5 + ) + return docs + + async def rag_fusion(self, query: str): + """Generate multiple queries for comprehensive retrieval""" + fusion_prompt = f"""Generate 3 different search queries for: {query} + 1. A specific technical query: + 2. A broader conceptual query: + 3. A related contextual query: + """ + + queries = await self.llm.ainvoke(fusion_prompt) + all_docs = [] + + for q in self._parse_queries(queries.content): + docs = await self.vectorstore.asimilarity_search(q, k=3) + all_docs.extend(docs) + + # Deduplicate and rerank + return self._deduplicate_docs(all_docs) +``` + +## Production Deployment Patterns + +### Async API Server with FastAPI +```python +from fastapi import FastAPI, HTTPException, BackgroundTasks +from fastapi.responses import StreamingResponse +from pydantic import BaseModel +import asyncio +from contextlib import asynccontextmanager + +class AgentRequest(BaseModel): + message: str + session_id: str + stream: bool = False + +class ProductionAgentServer: + def __init__(self): + self.agent = None + self.memory_store = {} + + @asynccontextmanager + async def lifespan(self, app: FastAPI): + # Startup: Initialize agent and resources + await self.initialize_agent() + yield + # Shutdown: Cleanup resources + await self.cleanup() + + async def initialize_agent(self): + """Initialize agent with all components""" + llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0, + streaming=True, + callbacks=[LangSmithCallbackHandler()] + ) + + tools = await self.setup_tools() + self.agent = create_react_agent(llm, tools) + + async def process_request(self, request: AgentRequest): + """Process agent request with session management""" + # Get or create session memory + memory = self.memory_store.get( + request.session_id, + ConversationTokenBufferMemory(max_token_limit=2000) + ) + + try: + if request.stream: + return StreamingResponse( + self._stream_response(request.message, memory), + media_type="text/event-stream" + ) + else: + result = await self.agent.ainvoke({ + "messages": [HumanMessage(content=request.message)], + "memory": memory + }) + return {"response": result["messages"][-1].content} + + except Exception as e: + logger.error(f"Agent error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + async def _stream_response(self, message: str, memory): + """Stream tokens as they're generated""" + async for chunk in self.agent.astream({ + "messages": [HumanMessage(content=message)], + "memory": memory + }): + if "messages" in chunk: + content = chunk["messages"][-1].content + yield f"data: {json.dumps({'token': content})}\n\n" + +# FastAPI app setup +app = FastAPI(lifespan=server.lifespan) +server = ProductionAgentServer() + +@app.post("/agent/invoke") +async def invoke_agent(request: AgentRequest): + return await server.process_request(request) +``` + +### Load Balancing & Scaling +```python +class AgentLoadBalancer: + def __init__(self, num_workers: int = 3): + self.workers = [] + self.current_worker = 0 + self.init_workers(num_workers) + + def init_workers(self, num_workers: int): + """Initialize multiple agent instances""" + for i in range(num_workers): + worker = { + "id": i, + "agent": self.create_agent_instance(), + "active_requests": 0, + "total_processed": 0 + } + self.workers.append(worker) + + async def route_request(self, request: dict): + """Route to least busy worker""" + # Find worker with minimum active requests + worker = min(self.workers, key=lambda w: w["active_requests"]) + + worker["active_requests"] += 1 + try: + result = await worker["agent"].ainvoke(request) + worker["total_processed"] += 1 + return result + finally: + worker["active_requests"] -= 1 +``` + +### Caching & Optimization +```python +from functools import lru_cache +import hashlib +import redis + +class AgentCacheManager: + def __init__(self): + self.redis_client = redis.Redis( + host='localhost', + port=6379, + decode_responses=True + ) + self.cache_ttl = 3600 # 1 hour + + def get_cache_key(self, query: str, context: dict) -> str: + """Generate deterministic cache key""" + cache_data = f"{query}_{json.dumps(context, sort_keys=True)}" + return hashlib.sha256(cache_data.encode()).hexdigest() + + async def get_cached_response(self, query: str, context: dict): + """Check for cached response""" + key = self.get_cache_key(query, context) + cached = self.redis_client.get(key) + + if cached: + logger.info(f"Cache hit for query: {query[:50]}...") + return json.loads(cached) + return None + + async def cache_response(self, query: str, context: dict, response: str): + """Cache the response""" + key = self.get_cache_key(query, context) + self.redis_client.setex( + key, + self.cache_ttl, + json.dumps(response) + ) +``` + +## Testing & Evaluation Strategies + +### Agent Testing Framework +```python +import pytest +from langchain.smith import RunEvalConfig +from langsmith import Client + +class AgentTestSuite: + def __init__(self, agent): + self.agent = agent + self.client = Client() + + @pytest.fixture + def test_cases(self): + return [ + { + "input": "What's the weather in NYC?", + "expected_tool": "weather_tool", + "validate_output": lambda x: "temperature" in x.lower() + }, + { + "input": "Calculate 25 * 4", + "expected_tool": "calculator", + "validate_output": lambda x: "100" in x + } + ] + + async def test_tool_selection(self, test_cases): + """Test if agent selects correct tools""" + for case in test_cases: + result = await self.agent.ainvoke({ + "messages": [HumanMessage(content=case["input"])] + }) + + # Check tool usage + tool_calls = self._extract_tool_calls(result) + assert case["expected_tool"] in tool_calls + + # Validate output + output = result["messages"][-1].content + assert case["validate_output"](output) + + async def test_error_handling(self): + """Test agent handles errors gracefully""" + # Simulate tool failure + with pytest.raises(Exception) as exc_info: + await self.agent.ainvoke({ + "messages": [HumanMessage(content="Use broken tool")], + "mock_tool_error": True + }) + + assert "gracefully handled" in str(exc_info.value) +``` + +### LangSmith Evaluation +```python +from langsmith.evaluation import evaluate + +class LangSmithEvaluator: + def __init__(self, dataset_name: str): + self.dataset_name = dataset_name + self.client = Client() + + async def run_evaluation(self, agent): + """Run comprehensive evaluation suite""" + eval_config = RunEvalConfig( + evaluators=[ + "qa", # Question-answering accuracy + "context_qa", # Retrieval relevance + "cot_qa", # Chain-of-thought reasoning + ], + custom_evaluators=[self.custom_evaluator], + eval_llm=ChatAnthropic(model="claude-sonnet-4-5", temperature=0) + ) + + results = await evaluate( + lambda inputs: agent.invoke({"messages": [HumanMessage(content=inputs["question"])]}), + data=self.dataset_name, + evaluators=eval_config, + experiment_prefix="agent_eval" + ) + + return results + + def custom_evaluator(self, run, example): + """Custom evaluation metrics""" + # Evaluate response quality + score = self._calculate_quality_score(run.outputs) + + return { + "score": score, + "key": "response_quality", + "comment": f"Quality score: {score:.2f}" + } +``` + +## Complete Code Examples + +### Example 1: Custom Multi-Tool Agent with Memory +```python +import os +from typing import List, Dict, Any +from langgraph.prebuilt import create_react_agent +from langchain_anthropic import ChatAnthropic +from langchain_core.tools import Tool +from langchain.memory import ConversationTokenBufferMemory +import asyncio +import numexpr # Safe math evaluation library + +class CustomMultiToolAgent: + def __init__(self): + # Initialize LLM + self.llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0, + streaming=True + ) + + # Initialize memory + self.memory = ConversationTokenBufferMemory( + llm=self.llm, + max_token_limit=2000, + return_messages=True + ) + + # Setup tools + self.tools = self._create_tools() + + # Create agent + self.agent = create_react_agent( + self.llm, + self.tools, + state_modifier="""You are a helpful AI assistant with access to multiple tools. + Use the tools to help answer questions accurately. + Always cite which tool you used for transparency.""" + ) + + def _create_tools(self) -> List[Tool]: + """Create custom tools for the agent""" + return [ + Tool( + name="calculator", + func=self._calculator, + description="Perform mathematical calculations" + ), + Tool( + name="web_search", + func=self._web_search, + description="Search the web for current information" + ), + Tool( + name="database_query", + func=self._database_query, + description="Query internal database for business data" + ) + ] + + async def _calculator(self, expression: str) -> str: + """Safe math evaluation using numexpr""" + try: + # Use numexpr for safe mathematical evaluation + # Only allows mathematical operations, no arbitrary code execution + result = numexpr.evaluate(expression) + return f"Result: {result}" + except Exception as e: + return f"Calculation error: {str(e)}" + + async def _web_search(self, query: str) -> str: + """Mock web search implementation""" + # Implement actual search API call + return f"Search results for '{query}': [mock results]" + + async def _database_query(self, query: str) -> str: + """Mock database query""" + # Implement actual database query + return f"Database results: [mock data]" + + async def process(self, user_input: str) -> str: + """Process user input and return response""" + # Add to memory + messages = self.memory.chat_memory.messages + + # Invoke agent + result = await self.agent.ainvoke({ + "messages": messages + [{"role": "human", "content": user_input}] + }) + + # Extract response + response = result["messages"][-1].content + + # Save to memory + self.memory.save_context( + {"input": user_input}, + {"output": response} + ) + + return response + +# Usage +async def main(): + agent = CustomMultiToolAgent() + + queries = [ + "What is 25 * 4 + 10?", + "Search for recent AI developments", + "What was my first question?" + ] + + for query in queries: + response = await agent.process(query) + print(f"Q: {query}\nA: {response}\n") + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Example 2: Production RAG Agent with Vector Store +```python +from langchain_voyageai import VoyageAIEmbeddings +from langchain_anthropic import ChatAnthropic +from langchain_pinecone import PineconeVectorStore +from langchain.chains import ConversationalRetrievalChain +from langchain.memory import ConversationSummaryBufferMemory +import pinecone +from typing import Optional + +class ProductionRAGAgent: + def __init__( + self, + index_name: str, + namespace: str = "default", + model: str = "claude-sonnet-4-5" + ): + # Initialize Pinecone + self.pc = pinecone.Pinecone(api_key=os.getenv("PINECONE_API_KEY")) + self.index = self.pc.Index(index_name) + + # Setup embeddings and LLM + # Using voyage-3-large - recommended by Anthropic for Claude Sonnet 4.5 + self.embeddings = VoyageAIEmbeddings(model="voyage-3-large") + self.llm = ChatAnthropic(model=model, temperature=0) + + # Initialize vector store + self.vectorstore = PineconeVectorStore( + index=self.index, + embedding=self.embeddings, + namespace=namespace + ) + + # Setup memory with summarization + self.memory = ConversationSummaryBufferMemory( + llm=self.llm, + max_token_limit=1000, + return_messages=True, + memory_key="chat_history", + output_key="answer" + ) + + # Create retrieval chain + self.chain = ConversationalRetrievalChain.from_llm( + llm=self.llm, + retriever=self.vectorstore.as_retriever( + search_type="similarity_score_threshold", + search_kwargs={ + "k": 5, + "score_threshold": 0.7 + } + ), + memory=self.memory, + return_source_documents=True, + verbose=True + ) + + async def ingest_document(self, file_path: str, chunk_size: int = 1000): + """Ingest and index a document""" + from langchain_community.document_loaders import PyPDFLoader + from langchain_text_splitters import RecursiveCharacterTextSplitter + + # Load document + loader = PyPDFLoader(file_path) + documents = await loader.aload() + + # Split into chunks + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=chunk_size, + chunk_overlap=200, + separators=["\n\n", "\n", ".", " "] + ) + chunks = text_splitter.split_documents(documents) + + # Add to vector store + texts = [chunk.page_content for chunk in chunks] + metadatas = [chunk.metadata for chunk in chunks] + + ids = await self.vectorstore.aadd_texts( + texts=texts, + metadatas=metadatas + ) + + return {"chunks_created": len(ids), "document": file_path} + + async def query( + self, + question: str, + filter_dict: Optional[Dict] = None + ) -> Dict[str, Any]: + """Query the RAG system""" + # Apply filters if provided + if filter_dict: + self.chain.retriever.search_kwargs["filter"] = filter_dict + + # Run query + result = await self.chain.ainvoke({"question": question}) + + # Format response + return { + "answer": result["answer"], + "sources": [ + { + "content": doc.page_content[:200] + "...", + "metadata": doc.metadata + } + for doc in result.get("source_documents", []) + ], + "chat_history": self.memory.chat_memory.messages[-10:] # Last 10 messages + } + + def clear_memory(self): + """Clear conversation memory""" + self.memory.clear() + +# Usage example +async def rag_example(): + agent = ProductionRAGAgent(index_name="knowledge-base") + + # Ingest documents + await agent.ingest_document("company_handbook.pdf") + + # Query the system + result = await agent.query("What is the company's remote work policy?") + print(f"Answer: {result['answer']}") + print(f"Sources: {result['sources']}") +``` + +### Example 3: Multi-Agent Orchestration System +```python +from langgraph.graph import StateGraph, MessagesState, START, END +from langgraph.types import Command +from typing import Literal, TypedDict, Annotated +from langchain_anthropic import ChatAnthropic +import json + +class ProjectState(TypedDict): + messages: Annotated[list, "conversation history"] + project_plan: Annotated[str, "project plan"] + code_implementation: Annotated[str, "implementation"] + test_results: Annotated[str, "test results"] + documentation: Annotated[str, "documentation"] + current_phase: Annotated[str, "current phase"] + +class MultiAgentOrchestrator: + def __init__(self): + self.llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0) + self.graph = self._build_graph() + + def _build_graph(self): + """Build the multi-agent workflow graph""" + builder = StateGraph(ProjectState) + + # Add agent nodes + builder.add_node("supervisor", self.supervisor_agent) + builder.add_node("planner", self.planner_agent) + builder.add_node("coder", self.coder_agent) + builder.add_node("tester", self.tester_agent) + builder.add_node("documenter", self.documenter_agent) + + # Add edges + builder.add_edge(START, "supervisor") + + # Supervisor routes to appropriate agent + builder.add_conditional_edges( + "supervisor", + self.route_supervisor, + { + "planner": "planner", + "coder": "coder", + "tester": "tester", + "documenter": "documenter", + "end": END + } + ) + + # Agents return to supervisor + builder.add_edge("planner", "supervisor") + builder.add_edge("coder", "supervisor") + builder.add_edge("tester", "supervisor") + builder.add_edge("documenter", "supervisor") + + return builder.compile() + + async def supervisor_agent(self, state: ProjectState) -> ProjectState: + """Supervisor decides next action""" + prompt = f""" + You are a project supervisor. Based on the current state, decide the next action. + + Current Phase: {state.get('current_phase', 'initial')} + Messages: {state['messages'][-1] if state['messages'] else 'No messages'} + + Decide which agent should work next or if the project is complete. + """ + + response = await self.llm.ainvoke(prompt) + + state["messages"].append({ + "role": "supervisor", + "content": response.content + }) + + return state + + def route_supervisor(self, state: ProjectState) -> Literal["planner", "coder", "tester", "documenter", "end"]: + """Route based on supervisor decision""" + last_message = state["messages"][-1]["content"] + + # Parse supervisor decision (implement actual parsing logic) + if "plan" in last_message.lower(): + return "planner" + elif "code" in last_message.lower(): + return "coder" + elif "test" in last_message.lower(): + return "tester" + elif "document" in last_message.lower(): + return "documenter" + else: + return "end" + + async def planner_agent(self, state: ProjectState) -> ProjectState: + """Planning agent creates project plan""" + prompt = f""" + Create a detailed implementation plan for: {state['messages'][0]['content']} + + Include: + 1. Architecture overview + 2. Component breakdown + 3. Implementation phases + 4. Testing strategy + """ + + plan = await self.llm.ainvoke(prompt) + state["project_plan"] = plan.content + state["current_phase"] = "planned" + + return state + + async def coder_agent(self, state: ProjectState) -> ProjectState: + """Coding agent implements the solution""" + prompt = f""" + Implement the following plan: + {state.get('project_plan', 'No plan available')} + + Write production-ready code with error handling. + """ + + code = await self.llm.ainvoke(prompt) + state["code_implementation"] = code.content + state["current_phase"] = "coded" + + return state + + async def tester_agent(self, state: ProjectState) -> ProjectState: + """Testing agent validates implementation""" + prompt = f""" + Review and test this implementation: + {state.get('code_implementation', 'No code available')} + + Provide test cases and results. + """ + + tests = await self.llm.ainvoke(prompt) + state["test_results"] = tests.content + state["current_phase"] = "tested" + + return state + + async def documenter_agent(self, state: ProjectState) -> ProjectState: + """Documentation agent creates docs""" + prompt = f""" + Create documentation for: + Plan: {state.get('project_plan', 'N/A')} + Code: {state.get('code_implementation', 'N/A')} + Tests: {state.get('test_results', 'N/A')} + """ + + docs = await self.llm.ainvoke(prompt) + state["documentation"] = docs.content + state["current_phase"] = "documented" + + return state + + async def execute_project(self, project_description: str): + """Execute the entire project workflow""" + initial_state = { + "messages": [{"role": "user", "content": project_description}], + "project_plan": "", + "code_implementation": "", + "test_results": "", + "documentation": "", + "current_phase": "initial" + } + + result = await self.graph.ainvoke(initial_state) + return result + +# Usage +async def orchestration_example(): + orchestrator = MultiAgentOrchestrator() + + result = await orchestrator.execute_project( + "Build a REST API for user authentication with JWT tokens" + ) + + print("Project Plan:", result["project_plan"]) + print("Implementation:", result["code_implementation"]) + print("Test Results:", result["test_results"]) + print("Documentation:", result["documentation"]) +``` + +### Example 4: Memory-Enhanced Conversational Agent +```python +from langchain.agents import create_tool_calling_agent, AgentExecutor +from langchain_anthropic import ChatAnthropic +from langchain.memory import ( + ConversationBufferMemory, + ConversationSummaryMemory, + ConversationEntityMemory, + CombinedMemory +) +from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder +from langgraph.checkpoint.memory import MemorySaver +import json + +class MemoryEnhancedAgent: + def __init__(self, session_id: str): + self.session_id = session_id + self.llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0.7) + + # Initialize multiple memory types + self.conversation_memory = ConversationBufferMemory( + memory_key="chat_history", + return_messages=True + ) + + self.summary_memory = ConversationSummaryMemory( + llm=self.llm, + memory_key="conversation_summary" + ) + + self.entity_memory = ConversationEntityMemory( + llm=self.llm, + memory_key="entities" + ) + + # Combine memories + self.combined_memory = CombinedMemory( + memories=[ + self.conversation_memory, + self.summary_memory, + self.entity_memory + ] + ) + + # Setup agent + self.agent = self._create_agent() + + def _create_agent(self): + """Create agent with memory-aware prompting""" + prompt = ChatPromptTemplate.from_messages([ + ("system", """You are a helpful AI assistant with perfect memory. + + Conversation Summary: + {conversation_summary} + + Known Entities: + {entities} + + Use this context to provide personalized, contextual responses. + Remember important details about the user and refer back to previous conversations. + """), + MessagesPlaceholder(variable_name="chat_history"), + ("human", "{input}"), + MessagesPlaceholder(variable_name="agent_scratchpad") + ]) + + tools = [] # Add your tools here + + agent = create_tool_calling_agent( + llm=self.llm, + tools=tools, + prompt=prompt + ) + + return AgentExecutor( + agent=agent, + tools=tools, + memory=self.combined_memory, + verbose=True, + return_intermediate_steps=True + ) + + async def chat(self, user_input: str) -> Dict[str, Any]: + """Process chat with full memory context""" + # Execute agent + result = await self.agent.ainvoke({"input": user_input}) + + # Extract entities for future reference + entities = self.entity_memory.entity_store.store + + # Get conversation summary + summary = self.summary_memory.buffer + + return { + "response": result["output"], + "entities": entities, + "summary": summary, + "session_id": self.session_id + } + + def save_session(self, filepath: str): + """Save session state to file""" + session_data = { + "session_id": self.session_id, + "chat_history": [ + {"role": m.type, "content": m.content} + for m in self.conversation_memory.chat_memory.messages + ], + "summary": self.summary_memory.buffer, + "entities": dict(self.entity_memory.entity_store.store) + } + + with open(filepath, 'w') as f: + json.dump(session_data, f, indent=2) + + def load_session(self, filepath: str): + """Load session state from file""" + with open(filepath, 'r') as f: + session_data = json.load(f) + + # Restore memories + # Implementation depends on specific memory types + self.session_id = session_data["session_id"] + + # Restore chat history + for msg in session_data["chat_history"]: + if msg["role"] == "human": + self.conversation_memory.chat_memory.add_user_message(msg["content"]) + else: + self.conversation_memory.chat_memory.add_ai_message(msg["content"]) + + # Restore summary + self.summary_memory.buffer = session_data["summary"] + + # Restore entities + for entity, info in session_data["entities"].items(): + self.entity_memory.entity_store.set(entity, info) + +# Usage example +async def memory_agent_example(): + agent = MemoryEnhancedAgent(session_id="user-123") + + # Conversation with memory + conversations = [ + "Hi, my name is Alice and I work at TechCorp", + "I'm interested in machine learning projects", + "What did I tell you about my work?", + "Can you remind me what we discussed about my interests?" + ] + + for msg in conversations: + result = await agent.chat(msg) + print(f"User: {msg}") + print(f"Agent: {result['response']}") + print(f"Entities tracked: {result['entities']}\n") + + # Save session + agent.save_session("session_user-123.json") +``` + +### Example 5: Production-Ready Deployment with Monitoring +```python +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from prometheus_client import Counter, Histogram, Gauge, generate_latest +import time +from langsmith import Client as LangSmithClient +from typing import Optional +import logging +from contextlib import asynccontextmanager + +# Metrics +request_count = Counter('agent_requests_total', 'Total agent requests') +request_duration = Histogram('agent_request_duration_seconds', 'Request duration') +active_sessions = Gauge('agent_active_sessions', 'Active agent sessions') +error_count = Counter('agent_errors_total', 'Total agent errors') + +class ProductionAgent: + def __init__(self): + self.langsmith_client = LangSmithClient() + self.agent = None + self.session_store = {} + + @asynccontextmanager + async def lifespan(self, app: FastAPI): + """Manage application lifecycle""" + # Startup + logging.info("Starting production agent...") + await self.initialize() + + yield + + # Shutdown + logging.info("Shutting down production agent...") + await self.cleanup() + + async def initialize(self): + """Initialize agent and dependencies""" + # Setup LLM + self.llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0) + + # Initialize agent with error handling + tools = await self.setup_tools_with_validation() + + self.agent = create_react_agent( + self.llm, + tools, + checkpointer=MemorySaver() # Enable conversation memory + ) + + async def setup_tools_with_validation(self): + """Setup and validate tools""" + tools = [] + + # Define tools with health checks + tool_configs = [ + {"name": "calculator", "func": self.calc_tool, "health_check": self.check_calc}, + {"name": "search", "func": self.search_tool, "health_check": self.check_search} + ] + + for config in tool_configs: + try: + # Run health check + await config["health_check"]() + + tools.append(Tool( + name=config["name"], + func=config["func"], + description=f"Tool: {config['name']}" + )) + + logging.info(f"Tool {config['name']} initialized successfully") + except Exception as e: + logging.error(f"Tool {config['name']} failed health check: {e}") + + return tools + + @request_duration.time() + async def process_request( + self, + message: str, + session_id: str, + timeout: float = 30.0 + ): + """Process request with monitoring and timeout""" + request_count.inc() + active_sessions.inc() + + try: + # Create timeout task + import asyncio + + task = asyncio.create_task( + self.agent.ainvoke( + {"messages": [{"role": "human", "content": message}]}, + config={"configurable": {"thread_id": session_id}} + ) + ) + + result = await asyncio.wait_for(task, timeout=timeout) + + # Log to LangSmith + self.langsmith_client.create_run( + name="agent_request", + inputs={"message": message, "session_id": session_id}, + outputs={"response": result["messages"][-1].content} + ) + + return { + "response": result["messages"][-1].content, + "session_id": session_id, + "latency": time.time() + } + + except asyncio.TimeoutError: + error_count.inc() + raise HTTPException(status_code=504, detail="Request timeout") + except Exception as e: + error_count.inc() + logging.error(f"Agent error: {e}") + raise HTTPException(status_code=500, detail=str(e)) + finally: + active_sessions.dec() + + async def health_check(self): + """Comprehensive health check""" + checks = { + "llm": False, + "tools": False, + "memory": False, + "langsmith": False + } + + try: + # Check LLM + test_response = await self.llm.ainvoke("test") + checks["llm"] = bool(test_response) + + # Check tools + checks["tools"] = len(await self.setup_tools_with_validation()) > 0 + + # Check memory store + checks["memory"] = self.session_store is not None + + # Check LangSmith connection + self.langsmith_client.list_projects(limit=1) + checks["langsmith"] = True + + except Exception as e: + logging.error(f"Health check failed: {e}") + + return { + "status": "healthy" if all(checks.values()) else "unhealthy", + "checks": checks, + "active_sessions": active_sessions._value.get(), + "total_requests": request_count._value.get() + } + +# FastAPI Application +agent_system = ProductionAgent() +app = FastAPI( + title="Production LangChain Agent", + version="1.0.0", + lifespan=agent_system.lifespan +) + +# Add CORS middleware +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"] +) + +@app.post("/chat") +async def chat(message: str, session_id: Optional[str] = None): + """Chat endpoint with session management""" + session_id = session_id or str(uuid.uuid4()) + return await agent_system.process_request(message, session_id) + +@app.get("/health") +async def health(): + """Health check endpoint""" + return await agent_system.health_check() + +@app.get("/metrics") +async def metrics(): + """Prometheus metrics endpoint""" + return generate_latest() + +if __name__ == "__main__": + import uvicorn + + # Run with production settings + uvicorn.run( + app, + host="0.0.0.0", + port=8000, + log_config="logging.yaml", + access_log=True, + use_colors=False + ) +``` + +## Reference Implementations + +### Reference 1: Enterprise Knowledge Assistant +```python +""" +Enterprise Knowledge Assistant with RAG, Memory, and Multi-Modal Support +Full implementation with production features +""" + +import os +from typing import List, Dict, Any, Optional +from dataclasses import dataclass +from enum import Enum + +# Core imports +from langchain_anthropic import ChatAnthropic +from langchain_voyageai import VoyageAIEmbeddings +from langgraph.graph import StateGraph, MessagesState, START, END +from langgraph.prebuilt import create_react_agent +from langgraph.checkpoint.postgres import PostgresSaver + +# Vector stores +from langchain_pinecone import PineconeVectorStore +from langchain_weaviate import WeaviateVectorStore + +# Memory +from langchain.memory import ConversationSummaryBufferMemory +from langchain.memory.chat_message_histories import RedisChatMessageHistory + +# Tools +from langchain_core.tools import Tool, StructuredTool +from langchain.tools.retriever import create_retriever_tool + +# Document processing +from langchain_community.document_loaders import PyPDFLoader, UnstructuredFileLoader +from langchain_text_splitters import RecursiveCharacterTextSplitter + +# Monitoring +from langsmith import Client as LangSmithClient +import structlog + +logger = structlog.get_logger() + +class QueryType(Enum): + FACTUAL = "factual" + ANALYTICAL = "analytical" + CREATIVE = "creative" + CONVERSATIONAL = "conversational" + +@dataclass +class EnterpriseConfig: + """Configuration for enterprise deployment""" + anthropic_api_key: str + voyage_api_key: str + pinecone_api_key: str + pinecone_environment: str + redis_url: str + postgres_url: str + langsmith_api_key: str + max_retries: int = 3 + timeout_seconds: int = 30 + cache_ttl: int = 3600 + +class EnterpriseKnowledgeAssistant: + """Production-ready enterprise knowledge assistant""" + + def __init__(self, config: EnterpriseConfig): + self.config = config + self.setup_llms() + self.setup_vector_stores() + self.setup_memory() + self.setup_monitoring() + self.agent = self.build_agent() + + def setup_llms(self): + """Setup LLM""" + self.llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0, + api_key=self.config.anthropic_api_key, + max_retries=self.config.max_retries + ) + + def setup_vector_stores(self): + """Setup multiple vector stores for different content types""" + import pinecone + + # Initialize Pinecone + pc = pinecone.Pinecone(api_key=self.config.pinecone_api_key) + + # Embeddings + # Using voyage-3-large for best retrieval quality with Claude Sonnet 4.5 + self.embeddings = VoyageAIEmbeddings( + model="voyage-3-large", + voyage_api_key=self.config.voyage_api_key + ) + + # Document store + self.doc_store = PineconeVectorStore( + index=pc.Index("enterprise-docs"), + embedding=self.embeddings, + namespace="documents" + ) + + # FAQ store + self.faq_store = PineconeVectorStore( + index=pc.Index("enterprise-faq"), + embedding=self.embeddings, + namespace="faqs" + ) + + def setup_memory(self): + """Setup distributed memory system""" + # Redis for message history + self.message_history = RedisChatMessageHistory( + session_id="default", + url=self.config.redis_url, + ttl=self.config.cache_ttl + ) + + # Summary memory + self.memory = ConversationSummaryBufferMemory( + llm=self.llm, + chat_memory=self.message_history, + max_token_limit=2000, + return_messages=True + ) + + # PostgreSQL checkpointer for state persistence + self.checkpointer = PostgresSaver.from_conn_string( + self.config.postgres_url + ) + + def setup_monitoring(self): + """Setup monitoring and observability""" + self.langsmith = LangSmithClient(api_key=self.config.langsmith_api_key) + + # Custom callbacks for monitoring + self.callbacks = [ + self.log_callback, + self.metrics_callback, + self.error_callback + ] + + def build_agent(self): + """Build the main agent with all components""" + # Create tools + tools = self.create_tools() + + # Build state graph + builder = StateGraph(MessagesState) + + # Add nodes + builder.add_node("classifier", self.classify_query) + builder.add_node("retriever", self.retrieve_context) + builder.add_node("agent", self.agent_node) + builder.add_node("validator", self.validate_response) + + # Add edges + builder.add_edge(START, "classifier") + builder.add_edge("classifier", "retriever") + builder.add_edge("retriever", "agent") + builder.add_edge("agent", "validator") + builder.add_edge("validator", END) + + # Compile with checkpointer + return builder.compile(checkpointer=self.checkpointer) + + def create_tools(self) -> List[Tool]: + """Create all agent tools""" + tools = [] + + # Document search tool + tools.append(create_retriever_tool( + self.doc_store.as_retriever(search_kwargs={"k": 5}), + "search_documents", + "Search internal company documents" + )) + + # FAQ search tool + tools.append(create_retriever_tool( + self.faq_store.as_retriever(search_kwargs={"k": 3}), + "search_faqs", + "Search frequently asked questions" + )) + + # Analytics tool + tools.append(StructuredTool.from_function( + func=self.analyze_data, + name="analyze_data", + description="Analyze business data and metrics" + )) + + # Email tool + tools.append(StructuredTool.from_function( + func=self.draft_email, + name="draft_email", + description="Draft professional emails" + )) + + return tools + + async def classify_query(self, state: MessagesState) -> MessagesState: + """Classify the type of query""" + query = state["messages"][-1].content + + classification_prompt = f""" + Classify this query into one of: factual, analytical, creative, conversational + Query: {query} + Classification: + """ + + result = await self.llm.ainvoke(classification_prompt) + query_type = self.parse_classification(result.content) + + state["query_type"] = query_type + logger.info("Query classified", query_type=query_type) + + return state + + async def retrieve_context(self, state: MessagesState) -> MessagesState: + """Retrieve relevant context based on query type""" + query = state["messages"][-1].content + query_type = state.get("query_type", QueryType.FACTUAL) + + contexts = [] + + if query_type in [QueryType.FACTUAL, QueryType.ANALYTICAL]: + # Search documents + doc_results = await self.doc_store.asimilarity_search(query, k=5) + contexts.extend([doc.page_content for doc in doc_results]) + + if query_type == QueryType.CONVERSATIONAL: + # Search FAQs + faq_results = await self.faq_store.asimilarity_search(query, k=3) + contexts.extend([doc.page_content for doc in faq_results]) + + state["context"] = "\n\n".join(contexts) + return state + + async def agent_node(self, state: MessagesState) -> MessagesState: + """Main agent processing node""" + context = state.get("context", "") + + # Build enhanced prompt with context + enhanced_prompt = f""" + Context Information: + {context} + + User Query: {state['messages'][-1].content} + + Provide a comprehensive answer using the context provided. + """ + + # Create agent with tools + agent = create_react_agent( + self.llm, + self.create_tools(), + state_modifier=enhanced_prompt + ) + + # Invoke agent + result = await agent.ainvoke(state) + + return result + + async def validate_response(self, state: MessagesState) -> MessagesState: + """Validate and potentially enhance response""" + response = state["messages"][-1].content + + # Check for hallucination + validation_prompt = f""" + Check if this response is grounded in the provided context: + Context: {state.get('context', 'No context')} + Response: {response} + + Is the response factual and grounded? (yes/no) + """ + + validation = await self.llm.ainvoke(validation_prompt) + + if "no" in validation.content.lower(): + # Regenerate with stricter grounding + logger.warning("Response failed validation, regenerating") + state["messages"][-1].content = "I need to verify that information. Let me search again..." + return await self.agent_node(state) + + return state + + async def analyze_data(self, query: str) -> str: + """Mock analytics tool""" + return f"Analytics results for: {query}" + + async def draft_email(self, subject: str, recipient: str, content: str) -> str: + """Mock email drafting tool""" + return f"Email draft to {recipient} about {subject}: {content}" + + def parse_classification(self, text: str) -> QueryType: + """Parse classification result""" + text_lower = text.lower() + for query_type in QueryType: + if query_type.value in text_lower: + return query_type + return QueryType.FACTUAL + + async def log_callback(self, event: Dict): + """Log events""" + logger.info("Agent event", **event) + + async def metrics_callback(self, event: Dict): + """Track metrics""" + # Implement metrics tracking + pass + + async def error_callback(self, error: Exception): + """Handle errors""" + logger.error("Agent error", error=str(error)) + + async def process(self, query: str, session_id: str) -> Dict[str, Any]: + """Main entry point for processing queries""" + try: + # Invoke agent + result = await self.agent.ainvoke( + {"messages": [{"role": "human", "content": query}]}, + config={"configurable": {"thread_id": session_id}} + ) + + # Extract response + response = result["messages"][-1].content + + # Log to LangSmith + self.langsmith.create_run( + name="enterprise_assistant", + inputs={"query": query, "session_id": session_id}, + outputs={"response": response} + ) + + return { + "response": response, + "session_id": session_id, + "sources": result.get("context", "") + } + + except Exception as e: + logger.error("Processing error", error=str(e)) + raise + +# Usage +async def main(): + config = EnterpriseConfig( + anthropic_api_key=os.getenv("ANTHROPIC_API_KEY"), + voyage_api_key=os.getenv("VOYAGE_API_KEY"), + pinecone_api_key=os.getenv("PINECONE_API_KEY"), + pinecone_environment="us-east-1", + redis_url="redis://localhost:6379", + postgres_url=os.getenv("DATABASE_URL"), + langsmith_api_key=os.getenv("LANGSMITH_API_KEY") + ) + + assistant = EnterpriseKnowledgeAssistant(config) + + # Process query + result = await assistant.process( + query="What is our company's remote work policy?", + session_id="user-123" + ) + + print(result) + +if __name__ == "__main__": + import asyncio + asyncio.run(main()) +``` + +### Reference 2: Autonomous Research Agent +```python +""" +Autonomous Research Agent with Web Search, Paper Analysis, and Report Generation +Complete implementation with multi-step reasoning +""" + +from typing import List, Dict, Any, Optional +from langgraph.graph import StateGraph, MessagesState, START, END +from langgraph.types import Command +from langchain_anthropic import ChatAnthropic +from langchain_core.tools import Tool +from langchain_community.utilities import GoogleSerperAPIWrapper +from langchain_community.document_loaders import ArxivLoader +import asyncio +from datetime import datetime + +class ResearchState(MessagesState): + """Extended state for research agent""" + research_query: str + search_results: List[Dict] + papers: List[Dict] + analysis: str + report: str + citations: List[str] + current_step: str + max_papers: int = 5 + +class AutonomousResearchAgent: + """Autonomous agent for conducting research and generating reports""" + + def __init__(self, anthropic_api_key: str, serper_api_key: str): + self.llm = ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0, + api_key=anthropic_api_key + ) + + self.search = GoogleSerperAPIWrapper( + serper_api_key=serper_api_key + ) + + self.graph = self.build_research_graph() + + def build_research_graph(self): + """Build the research workflow graph""" + builder = StateGraph(ResearchState) + + # Add research nodes + builder.add_node("planner", self.plan_research) + builder.add_node("searcher", self.search_web) + builder.add_node("paper_finder", self.find_papers) + builder.add_node("analyzer", self.analyze_content) + builder.add_node("synthesizer", self.synthesize_findings) + builder.add_node("report_writer", self.write_report) + builder.add_node("reviewer", self.review_report) + + # Define flow + builder.add_edge(START, "planner") + builder.add_edge("planner", "searcher") + builder.add_edge("searcher", "paper_finder") + builder.add_edge("paper_finder", "analyzer") + builder.add_edge("analyzer", "synthesizer") + builder.add_edge("synthesizer", "report_writer") + builder.add_edge("report_writer", "reviewer") + + # Conditional edge from reviewer + builder.add_conditional_edges( + "reviewer", + self.should_revise, + { + "revise": "report_writer", + "complete": END + } + ) + + return builder.compile() + + async def plan_research(self, state: ResearchState) -> ResearchState: + """Plan the research approach""" + query = state["messages"][-1].content + + planning_prompt = f""" + Create a research plan for: {query} + + Include: + 1. Key topics to investigate + 2. Types of sources needed + 3. Research methodology + 4. Expected deliverables + + Format as structured plan. + """ + + plan = await self.llm.ainvoke(planning_prompt) + + state["research_query"] = query + state["current_step"] = "planned" + state["messages"].append({ + "role": "assistant", + "content": f"Research plan created: {plan.content}" + }) + + return state + + async def search_web(self, state: ResearchState) -> ResearchState: + """Search web for relevant information""" + query = state["research_query"] + + # Perform multiple searches with different angles + search_queries = [ + query, + f"{query} recent developments 2024", + f"{query} research papers", + f"{query} industry applications" + ] + + all_results = [] + for sq in search_queries: + results = await asyncio.to_thread(self.search.run, sq) + all_results.append({ + "query": sq, + "results": results + }) + + state["search_results"] = all_results + state["current_step"] = "searched" + + return state + + async def find_papers(self, state: ResearchState) -> ResearchState: + """Find and download relevant research papers""" + query = state["research_query"] + + # Search arXiv for papers + arxiv_loader = ArxivLoader( + query=query, + load_max_docs=state["max_papers"] + ) + + papers = await asyncio.to_thread(arxiv_loader.load) + + # Process papers + processed_papers = [] + for paper in papers: + processed_papers.append({ + "title": paper.metadata.get("Title", "Unknown"), + "authors": paper.metadata.get("Authors", "Unknown"), + "summary": paper.metadata.get("Summary", "")[:500], + "content": paper.page_content[:1000], # First 1000 chars + "arxiv_id": paper.metadata.get("Entry ID", "") + }) + + state["papers"] = processed_papers + state["current_step"] = "papers_found" + + return state + + async def analyze_content(self, state: ResearchState) -> ResearchState: + """Analyze all gathered content""" + search_results = state["search_results"] + papers = state["papers"] + + analysis_prompt = f""" + Analyze the following research materials: + + Web Search Results: + {search_results} + + Academic Papers: + {papers} + + Provide: + 1. Key findings and insights + 2. Common themes and patterns + 3. Contradictions or debates + 4. Knowledge gaps + 5. Practical implications + """ + + analysis = await self.llm.ainvoke(analysis_prompt) + + state["analysis"] = analysis.content + state["current_step"] = "analyzed" + + return state + + async def synthesize_findings(self, state: ResearchState) -> ResearchState: + """Synthesize all findings into coherent insights""" + analysis = state["analysis"] + + synthesis_prompt = f""" + Synthesize the following analysis into key insights: + + {analysis} + + Create: + 1. Executive summary (3-5 sentences) + 2. Main conclusions (bullet points) + 3. Recommendations + 4. Future research directions + """ + + synthesis = await self.llm.ainvoke(synthesis_prompt) + + state["messages"].append({ + "role": "assistant", + "content": synthesis.content + }) + state["current_step"] = "synthesized" + + return state + + async def write_report(self, state: ResearchState) -> ResearchState: + """Write comprehensive research report""" + query = state["research_query"] + analysis = state["analysis"] + papers = state["papers"] + + report_prompt = f""" + Write a comprehensive research report on: {query} + + Based on analysis: {analysis} + + Structure: + 1. Executive Summary + 2. Introduction + 3. Methodology + 4. Key Findings + 5. Discussion + 6. Conclusions + 7. References + + Include citations to papers: {[p['title'] for p in papers]} + + Make it professional and well-structured. + """ + + report = await self.llm.ainvoke(report_prompt) + + # Generate citations + citations = [] + for paper in papers: + citation = f"{paper['authors']} ({datetime.now().year}). {paper['title']}. arXiv:{paper['arxiv_id']}" + citations.append(citation) + + state["report"] = report.content + state["citations"] = citations + state["current_step"] = "report_written" + + return state + + async def review_report(self, state: ResearchState) -> ResearchState: + """Review and validate the report""" + report = state["report"] + + review_prompt = f""" + Review this research report for: + 1. Accuracy and factual correctness + 2. Logical flow and structure + 3. Completeness + 4. Professional tone + 5. Proper citations + + Report: + {report} + + Provide a quality score (1-10) and identify any issues. + """ + + review = await self.llm.ainvoke(review_prompt) + + state["messages"].append({ + "role": "assistant", + "content": f"Report review: {review.content}" + }) + + # Parse quality score + try: + import re + score_match = re.search(r'\b([1-9]|10)\b', review.content) + quality_score = int(score_match.group()) if score_match else 7 + except: + quality_score = 7 + + state["quality_score"] = quality_score + state["current_step"] = "reviewed" + + return state + + def should_revise(self, state: ResearchState) -> str: + """Decide whether to revise the report""" + quality_score = state.get("quality_score", 7) + + if quality_score < 7: + return "revise" + return "complete" + + async def conduct_research(self, topic: str) -> Dict[str, Any]: + """Main entry point for conducting research""" + initial_state = { + "messages": [{"role": "human", "content": topic}], + "research_query": "", + "search_results": [], + "papers": [], + "analysis": "", + "report": "", + "citations": [], + "current_step": "initial", + "max_papers": 5 + } + + result = await self.graph.ainvoke(initial_state) + + return { + "report": result["report"], + "citations": result["citations"], + "quality_score": result.get("quality_score", 0), + "steps_completed": result["current_step"] + } + +# Usage example +async def research_example(): + agent = AutonomousResearchAgent( + anthropic_api_key=os.getenv("ANTHROPIC_API_KEY"), + serper_api_key=os.getenv("SERPER_API_KEY") + ) + + result = await agent.conduct_research( + "Recent advances in quantum computing and their applications in cryptography" + ) + + print("Research Report:") + print(result["report"]) + print("\nCitations:") + for citation in result["citations"]: + print(f"- {citation}") + print(f"\nQuality Score: {result['quality_score']}/10") +``` + +### Reference 3: Real-time Collaborative Agent System +```python +""" +Real-time Collaborative Multi-Agent System with WebSocket Support +Production implementation with agent coordination and live updates +""" + +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse +import json +import asyncio +from typing import Dict, List, Set, Any +from datetime import datetime +from langgraph.graph import StateGraph, MessagesState +from langchain_anthropic import ChatAnthropic +import redis.asyncio as redis +from collections import defaultdict + +class CollaborativeAgentSystem: + """Real-time collaborative agent system with WebSocket support""" + + def __init__(self): + self.app = FastAPI() + self.setup_routes() + self.active_connections: Dict[str, Set[WebSocket]] = defaultdict(set) + self.agent_pool = {} + self.redis_client = None + self.llm = ChatAnthropic(model="claude-sonnet-4-5", temperature=0.7) + + async def startup(self): + """Initialize system resources""" + self.redis_client = await redis.from_url("redis://localhost:6379") + await self.initialize_agents() + + async def shutdown(self): + """Cleanup resources""" + if self.redis_client: + await self.redis_client.close() + + async def initialize_agents(self): + """Initialize specialized agents""" + agent_configs = [ + {"id": "coordinator", "role": "Project Coordinator", "specialty": "task planning"}, + {"id": "developer", "role": "Senior Developer", "specialty": "code implementation"}, + {"id": "reviewer", "role": "Code Reviewer", "specialty": "quality assurance"}, + {"id": "documenter", "role": "Technical Writer", "specialty": "documentation"} + ] + + for config in agent_configs: + self.agent_pool[config["id"]] = self.create_specialized_agent(config) + + def create_specialized_agent(self, config: Dict) -> Dict: + """Create a specialized agent with specific capabilities""" + return { + "id": config["id"], + "role": config["role"], + "specialty": config["specialty"], + "llm": ChatAnthropic( + model="claude-sonnet-4-5", + temperature=0.3 + ), + "status": "idle", + "current_task": None + } + + def setup_routes(self): + """Setup WebSocket and HTTP routes""" + + @self.app.websocket("/ws/{session_id}") + async def websocket_endpoint(websocket: WebSocket, session_id: str): + await self.handle_websocket(websocket, session_id) + + @self.app.post("/session/{session_id}/task") + async def create_task(session_id: str, task: Dict): + return await self.process_task(session_id, task) + + @self.app.get("/session/{session_id}/status") + async def get_status(session_id: str): + return await self.get_session_status(session_id) + + async def handle_websocket(self, websocket: WebSocket, session_id: str): + """Handle WebSocket connections for real-time updates""" + await websocket.accept() + self.active_connections[session_id].add(websocket) + + try: + # Send initial status + await websocket.send_json({ + "type": "connection", + "session_id": session_id, + "agents": list(self.agent_pool.keys()), + "timestamp": datetime.now().isoformat() + }) + + # Handle incoming messages + while True: + data = await websocket.receive_json() + await self.handle_client_message(session_id, data, websocket) + + except WebSocketDisconnect: + self.active_connections[session_id].remove(websocket) + if not self.active_connections[session_id]: + del self.active_connections[session_id] + + async def handle_client_message(self, session_id: str, data: Dict, websocket: WebSocket): + """Process messages from clients""" + message_type = data.get("type") + + if message_type == "task": + await self.distribute_task(session_id, data["content"]) + elif message_type == "chat": + await self.handle_chat(session_id, data["content"], data.get("agent_id")) + elif message_type == "command": + await self.handle_command(session_id, data["command"], data.get("args")) + + async def distribute_task(self, session_id: str, task_description: str): + """Distribute task among agents""" + # Coordinator analyzes and breaks down the task + coordinator = self.agent_pool["coordinator"] + + breakdown_prompt = f""" + Break down this task into subtasks for the team: + Task: {task_description} + + Available agents: + - Developer: code implementation + - Reviewer: quality assurance + - Documenter: documentation + + Provide a structured plan with assigned agents. + """ + + plan = await coordinator["llm"].ainvoke(breakdown_prompt) + + # Broadcast plan to all connected clients + await self.broadcast_to_session(session_id, { + "type": "plan", + "agent": "coordinator", + "content": plan.content, + "timestamp": datetime.now().isoformat() + }) + + # Execute subtasks in parallel + subtasks = self.parse_subtasks(plan.content) + results = await asyncio.gather(*[ + self.execute_subtask(session_id, subtask) + for subtask in subtasks + ]) + + # Aggregate results + await self.aggregate_results(session_id, results) + + def parse_subtasks(self, plan_content: str) -> List[Dict]: + """Parse subtasks from plan""" + # Simplified parsing - in production use structured output + subtasks = [] + + if "developer" in plan_content.lower(): + subtasks.append({ + "agent_id": "developer", + "task": "Implement the required functionality" + }) + + if "reviewer" in plan_content.lower(): + subtasks.append({ + "agent_id": "reviewer", + "task": "Review the implementation" + }) + + if "documenter" in plan_content.lower(): + subtasks.append({ + "agent_id": "documenter", + "task": "Create documentation" + }) + + return subtasks + + async def execute_subtask(self, session_id: str, subtask: Dict) -> Dict: + """Execute a subtask with a specific agent""" + agent_id = subtask["agent_id"] + agent = self.agent_pool[agent_id] + + # Update agent status + agent["status"] = "working" + agent["current_task"] = subtask["task"] + + # Broadcast status update + await self.broadcast_to_session(session_id, { + "type": "agent_status", + "agent": agent_id, + "status": "working", + "task": subtask["task"], + "timestamp": datetime.now().isoformat() + }) + + # Execute task + try: + result = await agent["llm"].ainvoke(subtask["task"]) + + # Store result in Redis + await self.redis_client.hset( + f"session:{session_id}:results", + agent_id, + json.dumps({ + "content": result.content, + "timestamp": datetime.now().isoformat() + }) + ) + + # Broadcast completion + await self.broadcast_to_session(session_id, { + "type": "task_complete", + "agent": agent_id, + "result": result.content, + "timestamp": datetime.now().isoformat() + }) + + return { + "agent_id": agent_id, + "result": result.content, + "success": True + } + + except Exception as e: + await self.broadcast_to_session(session_id, { + "type": "error", + "agent": agent_id, + "error": str(e), + "timestamp": datetime.now().isoformat() + }) + + return { + "agent_id": agent_id, + "error": str(e), + "success": False + } + + finally: + # Reset agent status + agent["status"] = "idle" + agent["current_task"] = None + + async def aggregate_results(self, session_id: str, results: List[Dict]): + """Aggregate results from all agents""" + coordinator = self.agent_pool["coordinator"] + + summary_prompt = f""" + Aggregate and summarize the following results from the team: + + {json.dumps(results, indent=2)} + + Provide a cohesive summary of the completed work. + """ + + summary = await coordinator["llm"].ainvoke(summary_prompt) + + # Broadcast final summary + await self.broadcast_to_session(session_id, { + "type": "final_summary", + "agent": "coordinator", + "content": summary.content, + "timestamp": datetime.now().isoformat() + }) + + async def handle_chat(self, session_id: str, message: str, agent_id: Optional[str] = None): + """Handle chat messages directed at specific agents""" + if agent_id and agent_id in self.agent_pool: + agent = self.agent_pool[agent_id] + response = await agent["llm"].ainvoke(message) + + await self.broadcast_to_session(session_id, { + "type": "chat_response", + "agent": agent_id, + "content": response.content, + "timestamp": datetime.now().isoformat() + }) + else: + # Broadcast to all agents and get responses + responses = await asyncio.gather(*[ + agent["llm"].ainvoke(message) + for agent in self.agent_pool.values() + ]) + + for agent_id, response in zip(self.agent_pool.keys(), responses): + await self.broadcast_to_session(session_id, { + "type": "chat_response", + "agent": agent_id, + "content": response.content, + "timestamp": datetime.now().isoformat() + }) + + async def handle_command(self, session_id: str, command: str, args: Dict): + """Handle system commands""" + if command == "reset": + await self.reset_session(session_id) + elif command == "export": + await self.export_session(session_id) + elif command == "pause": + await self.pause_agents(session_id) + elif command == "resume": + await self.resume_agents(session_id) + + async def broadcast_to_session(self, session_id: str, message: Dict): + """Broadcast message to all connections in a session""" + if session_id in self.active_connections: + disconnected = set() + + for websocket in self.active_connections[session_id]: + try: + await websocket.send_json(message) + except: + disconnected.add(websocket) + + # Clean up disconnected websockets + for ws in disconnected: + self.active_connections[session_id].remove(ws) + + async def get_session_status(self, session_id: str) -> Dict: + """Get current session status""" + agent_statuses = { + agent_id: { + "status": agent["status"], + "current_task": agent["current_task"] + } + for agent_id, agent in self.agent_pool.items() + } + + # Get results from Redis + results = await self.redis_client.hgetall(f"session:{session_id}:results") + + return { + "session_id": session_id, + "agents": agent_statuses, + "results": { + k.decode(): json.loads(v.decode()) + for k, v in results.items() + } if results else {}, + "active_connections": len(self.active_connections.get(session_id, set())), + "timestamp": datetime.now().isoformat() + } + + async def reset_session(self, session_id: str): + """Reset session state""" + # Clear Redis data + await self.redis_client.delete(f"session:{session_id}:results") + + # Reset agents + for agent in self.agent_pool.values(): + agent["status"] = "idle" + agent["current_task"] = None + + await self.broadcast_to_session(session_id, { + "type": "system", + "message": "Session reset", + "timestamp": datetime.now().isoformat() + }) + + async def export_session(self, session_id: str) -> Dict: + """Export session data""" + results = await self.redis_client.hgetall(f"session:{session_id}:results") + + export_data = { + "session_id": session_id, + "timestamp": datetime.now().isoformat(), + "results": { + k.decode(): json.loads(v.decode()) + for k, v in results.items() + } if results else {} + } + + return export_data + +# Create application instance +collab_system = CollaborativeAgentSystem() +app = collab_system.app + +# Add startup and shutdown events +@app.on_event("startup") +async def startup_event(): + await collab_system.startup() + +@app.on_event("shutdown") +async def shutdown_event(): + await collab_system.shutdown() + +# HTML client for testing +@app.get("/") +async def get(): + return HTMLResponse(""" + + + + Collaborative Agent System + + +

Collaborative Agent System

+
+ + + + + + + """) + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) +``` + +## Summary + +This comprehensive LangChain/LangGraph agent development guide provides: + +1. **Modern Architecture Patterns**: State-based agent orchestration with LangGraph +2. **Production-Ready Components**: Async patterns, error handling, monitoring +3. **Advanced Memory Systems**: Multiple memory types with distributed storage +4. **RAG Integration**: Vector stores, reranking, and hybrid search +5. **Multi-Agent Coordination**: Specialized agents working together +6. **Real-time Capabilities**: WebSocket support for live updates +7. **Enterprise Features**: Security, scalability, and observability +8. **Complete Examples**: Full implementations ready for production use + +The guide emphasizes production reliability, scalability, and maintainability while leveraging the latest LangChain 0.1+ and LangGraph capabilities for building sophisticated AI agent systems. \ No newline at end of file diff --git a/tools/monitor-setup.md b/tools/monitor-setup.md index 771afc4..27fb725 100644 --- a/tools/monitor-setup.md +++ b/tools/monitor-setup.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Monitoring and Observability Setup You are a monitoring and observability expert specializing in implementing comprehensive monitoring solutions. Set up metrics collection, distributed tracing, log aggregation, and create insightful dashboards that provide full visibility into system health and performance. diff --git a/tools/multi-agent-optimize.md b/tools/multi-agent-optimize.md index 5499c33..dc4f12a 100644 --- a/tools/multi-agent-optimize.md +++ b/tools/multi-agent-optimize.md @@ -1,90 +1,189 @@ ---- -model: sonnet ---- +# Multi-Agent Optimization Toolkit -Optimize application stack using specialized optimization agents: +## Role: AI-Powered Multi-Agent Performance Engineering Specialist -[Extended thinking: This tool coordinates database, performance, and frontend optimization agents to improve application performance holistically. Each agent focuses on their domain while ensuring optimizations work together.] +### Context +The Multi-Agent Optimization Tool is an advanced AI-driven framework designed to holistically improve system performance through intelligent, coordinated agent-based optimization. Leveraging cutting-edge AI orchestration techniques, this tool provides a comprehensive approach to performance engineering across multiple domains. -## Optimization Strategy +### Core Capabilities +- Intelligent multi-agent coordination +- Performance profiling and bottleneck identification +- Adaptive optimization strategies +- Cross-domain performance optimization +- Cost and efficiency tracking -### 1. Database Optimization -Use Task tool with subagent_type="database-optimizer" to: -- Analyze query performance and execution plans -- Optimize indexes and table structures -- Implement caching strategies -- Review connection pooling and configurations -- Suggest schema improvements +## Arguments Handling +The tool processes optimization arguments with flexible input parameters: +- `$TARGET`: Primary system/application to optimize +- `$PERFORMANCE_GOALS`: Specific performance metrics and objectives +- `$OPTIMIZATION_SCOPE`: Depth of optimization (quick-win, comprehensive) +- `$BUDGET_CONSTRAINTS`: Cost and resource limitations +- `$QUALITY_METRICS`: Performance quality thresholds -Prompt: "Optimize database layer for: $ARGUMENTS. Analyze and improve: -1. Slow query identification and optimization -2. Index analysis and recommendations -3. Schema optimization for performance -4. Connection pool tuning -5. Caching strategy implementation" +## 1. Multi-Agent Performance Profiling -### 2. Application Performance -Use Task tool with subagent_type="performance-engineer" to: -- Profile application code -- Identify CPU and memory bottlenecks -- Optimize algorithms and data structures -- Implement caching at application level -- Improve async/concurrent operations +### Profiling Strategy +- Distributed performance monitoring across system layers +- Real-time metrics collection and analysis +- Continuous performance signature tracking -Prompt: "Optimize application performance for: $ARGUMENTS. Focus on: -1. Code profiling and bottleneck identification -2. Algorithm optimization -3. Memory usage optimization -4. Concurrency improvements -5. Application-level caching" +#### Profiling Agents +1. **Database Performance Agent** + - Query execution time analysis + - Index utilization tracking + - Resource consumption monitoring -### 3. Frontend Optimization -Use Task tool with subagent_type="frontend-developer" to: -- Reduce bundle sizes -- Implement lazy loading -- Optimize rendering performance -- Improve Core Web Vitals -- Implement efficient state management +2. **Application Performance Agent** + - CPU and memory profiling + - Algorithmic complexity assessment + - Concurrency and async operation analysis -Prompt: "Optimize frontend performance for: $ARGUMENTS. Improve: -1. Bundle size reduction strategies -2. Lazy loading implementation -3. Rendering optimization -4. Core Web Vitals (LCP, FID, CLS) -5. Network request optimization" +3. **Frontend Performance Agent** + - Rendering performance metrics + - Network request optimization + - Core Web Vitals monitoring -## Consolidated Optimization Plan +### Profiling Code Example +```python +def multi_agent_profiler(target_system): + agents = [ + DatabasePerformanceAgent(target_system), + ApplicationPerformanceAgent(target_system), + FrontendPerformanceAgent(target_system) + ] -### Performance Baseline -- Current performance metrics -- Identified bottlenecks -- User experience impact + performance_profile = {} + for agent in agents: + performance_profile[agent.__class__.__name__] = agent.profile() -### Optimization Roadmap -1. **Quick Wins** (< 1 day) - - Simple query optimizations - - Basic caching implementation - - Bundle splitting + return aggregate_performance_metrics(performance_profile) +``` -2. **Medium Improvements** (1-3 days) - - Index optimization - - Algorithm improvements - - Lazy loading implementation +## 2. Context Window Optimization -3. **Major Optimizations** (3+ days) - - Schema redesign - - Architecture changes - - Full caching layer +### Optimization Techniques +- Intelligent context compression +- Semantic relevance filtering +- Dynamic context window resizing +- Token budget management -### Expected Improvements -- Database query time reduction: X% -- API response time improvement: X% -- Frontend load time reduction: X% -- Overall user experience impact +### Context Compression Algorithm +```python +def compress_context(context, max_tokens=4000): + # Semantic compression using embedding-based truncation + compressed_context = semantic_truncate( + context, + max_tokens=max_tokens, + importance_threshold=0.7 + ) + return compressed_context +``` -### Implementation Priority -- Ordered list of optimizations by impact/effort ratio -- Dependencies between optimizations -- Risk assessment for each change +## 3. Agent Coordination Efficiency -Target for optimization: $ARGUMENTS \ No newline at end of file +### Coordination Principles +- Parallel execution design +- Minimal inter-agent communication overhead +- Dynamic workload distribution +- Fault-tolerant agent interactions + +### Orchestration Framework +```python +class MultiAgentOrchestrator: + def __init__(self, agents): + self.agents = agents + self.execution_queue = PriorityQueue() + self.performance_tracker = PerformanceTracker() + + def optimize(self, target_system): + # Parallel agent execution with coordinated optimization + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = { + executor.submit(agent.optimize, target_system): agent + for agent in self.agents + } + + for future in concurrent.futures.as_completed(futures): + agent = futures[future] + result = future.result() + self.performance_tracker.log(agent, result) +``` + +## 4. Parallel Execution Optimization + +### Key Strategies +- Asynchronous agent processing +- Workload partitioning +- Dynamic resource allocation +- Minimal blocking operations + +## 5. Cost Optimization Strategies + +### LLM Cost Management +- Token usage tracking +- Adaptive model selection +- Caching and result reuse +- Efficient prompt engineering + +### Cost Tracking Example +```python +class CostOptimizer: + def __init__(self): + self.token_budget = 100000 # Monthly budget + self.token_usage = 0 + self.model_costs = { + 'gpt-4': 0.03, + 'claude-3-sonnet': 0.015, + 'claude-3-haiku': 0.0025 + } + + def select_optimal_model(self, complexity): + # Dynamic model selection based on task complexity and budget + pass +``` + +## 6. Latency Reduction Techniques + +### Performance Acceleration +- Predictive caching +- Pre-warming agent contexts +- Intelligent result memoization +- Reduced round-trip communication + +## 7. Quality vs Speed Tradeoffs + +### Optimization Spectrum +- Performance thresholds +- Acceptable degradation margins +- Quality-aware optimization +- Intelligent compromise selection + +## 8. Monitoring and Continuous Improvement + +### Observability Framework +- Real-time performance dashboards +- Automated optimization feedback loops +- Machine learning-driven improvement +- Adaptive optimization strategies + +## Reference Workflows + +### Workflow 1: E-Commerce Platform Optimization +1. Initial performance profiling +2. Agent-based optimization +3. Cost and performance tracking +4. Continuous improvement cycle + +### Workflow 2: Enterprise API Performance Enhancement +1. Comprehensive system analysis +2. Multi-layered agent optimization +3. Iterative performance refinement +4. Cost-efficient scaling strategy + +## Key Considerations +- Always measure before and after optimization +- Maintain system stability during optimization +- Balance performance gains with resource consumption +- Implement gradual, reversible changes + +Target Optimization: $ARGUMENTS \ No newline at end of file diff --git a/tools/multi-agent-review.md b/tools/multi-agent-review.md index 2f59a86..8b37727 100644 --- a/tools/multi-agent-review.md +++ b/tools/multi-agent-review.md @@ -1,68 +1,194 @@ ---- -model: sonnet ---- +# Multi-Agent Code Review Orchestration Tool -Perform comprehensive multi-agent code review with specialized reviewers: +## Role: Expert Multi-Agent Review Orchestration Specialist -[Extended thinking: This tool command invokes multiple review-focused agents to provide different perspectives on code quality, security, and architecture. Each agent reviews independently, then findings are consolidated.] +A sophisticated AI-powered code review system designed to provide comprehensive, multi-perspective analysis of software artifacts through intelligent agent coordination and specialized domain expertise. -## Review Process +## Context and Purpose -### 1. Code Quality Review -Use Task tool with subagent_type="code-reviewer" to examine: -- Code style and readability -- Adherence to SOLID principles -- Design patterns and anti-patterns -- Code duplication and complexity -- Documentation completeness -- Test coverage and quality +The Multi-Agent Review Tool leverages a distributed, specialized agent network to perform holistic code assessments that transcend traditional single-perspective review approaches. By coordinating agents with distinct expertise, we generate a comprehensive evaluation that captures nuanced insights across multiple critical dimensions: -Prompt: "Perform detailed code review of: $ARGUMENTS. Focus on maintainability, readability, and best practices. Provide specific line-by-line feedback where appropriate." +- **Depth**: Specialized agents dive deep into specific domains +- **Breadth**: Parallel processing enables comprehensive coverage +- **Intelligence**: Context-aware routing and intelligent synthesis +- **Adaptability**: Dynamic agent selection based on code characteristics -### 2. Security Review -Use Task tool with subagent_type="security-auditor" to check: -- Authentication and authorization flaws -- Input validation and sanitization -- SQL injection and XSS vulnerabilities -- Sensitive data exposure -- Security misconfigurations -- Dependency vulnerabilities +## Tool Arguments and Configuration -Prompt: "Conduct security review of: $ARGUMENTS. Identify vulnerabilities, security risks, and OWASP compliance issues. Provide severity ratings and remediation steps." +### Input Parameters +- `$ARGUMENTS`: Target code/project for review + - Supports: File paths, Git repositories, code snippets + - Handles multiple input formats + - Enables context extraction and agent routing -### 3. Architecture Review -Use Task tool with subagent_type="architect-reviewer" to evaluate: -- Service boundaries and coupling -- Scalability considerations -- Design pattern appropriateness -- Technology choices -- API design quality -- Data flow and dependencies +### Agent Types +1. Code Quality Reviewers +2. Security Auditors +3. Architecture Specialists +4. Performance Analysts +5. Compliance Validators +6. Best Practices Experts -Prompt: "Review architecture and design of: $ARGUMENTS. Evaluate scalability, maintainability, and architectural patterns. Identify potential bottlenecks and design improvements." +## Multi-Agent Coordination Strategy -## Consolidated Review Output +### 1. Agent Selection and Routing Logic +- **Dynamic Agent Matching**: + - Analyze input characteristics + - Select most appropriate agent types + - Configure specialized sub-agents dynamically +- **Expertise Routing**: + ```python + def route_agents(code_context): + agents = [] + if is_web_application(code_context): + agents.extend([ + "security-auditor", + "web-architecture-reviewer" + ]) + if is_performance_critical(code_context): + agents.append("performance-analyst") + return agents + ``` -After all agents complete their reviews, consolidate findings into: +### 2. Context Management and State Passing +- **Contextual Intelligence**: + - Maintain shared context across agent interactions + - Pass refined insights between agents + - Support incremental review refinement +- **Context Propagation Model**: + ```python + class ReviewContext: + def __init__(self, target, metadata): + self.target = target + self.metadata = metadata + self.agent_insights = {} -1. **Critical Issues** - Must fix before merge - - Security vulnerabilities - - Broken functionality - - Major architectural flaws + def update_insights(self, agent_type, insights): + self.agent_insights[agent_type] = insights + ``` -2. **Important Issues** - Should fix soon - - Performance problems - - Code quality issues - - Missing tests +### 3. Parallel vs Sequential Execution +- **Hybrid Execution Strategy**: + - Parallel execution for independent reviews + - Sequential processing for dependent insights + - Intelligent timeout and fallback mechanisms +- **Execution Flow**: + ```python + def execute_review(review_context): + # Parallel independent agents + parallel_agents = [ + "code-quality-reviewer", + "security-auditor" + ] -3. **Minor Issues** - Nice to fix - - Style inconsistencies - - Documentation gaps - - Refactoring opportunities + # Sequential dependent agents + sequential_agents = [ + "architecture-reviewer", + "performance-optimizer" + ] + ``` -4. **Positive Findings** - Good practices to highlight - - Well-designed components - - Good test coverage - - Security best practices +### 4. Result Aggregation and Synthesis +- **Intelligent Consolidation**: + - Merge insights from multiple agents + - Resolve conflicting recommendations + - Generate unified, prioritized report +- **Synthesis Algorithm**: + ```python + def synthesize_review_insights(agent_results): + consolidated_report = { + "critical_issues": [], + "important_issues": [], + "improvement_suggestions": [] + } + # Intelligent merging logic + return consolidated_report + ``` + +### 5. Conflict Resolution Mechanism +- **Smart Conflict Handling**: + - Detect contradictory agent recommendations + - Apply weighted scoring + - Escalate complex conflicts +- **Resolution Strategy**: + ```python + def resolve_conflicts(agent_insights): + conflict_resolver = ConflictResolutionEngine() + return conflict_resolver.process(agent_insights) + ``` + +### 6. Performance Optimization +- **Efficiency Techniques**: + - Minimal redundant processing + - Cached intermediate results + - Adaptive agent resource allocation +- **Optimization Approach**: + ```python + def optimize_review_process(review_context): + return ReviewOptimizer.allocate_resources(review_context) + ``` + +### 7. Quality Validation Framework +- **Comprehensive Validation**: + - Cross-agent result verification + - Statistical confidence scoring + - Continuous learning and improvement +- **Validation Process**: + ```python + def validate_review_quality(review_results): + quality_score = QualityScoreCalculator.compute(review_results) + return quality_score > QUALITY_THRESHOLD + ``` + +## Example Implementations + +### 1. Parallel Code Review Scenario +```python +multi_agent_review( + target="/path/to/project", + agents=[ + {"type": "security-auditor", "weight": 0.3}, + {"type": "architecture-reviewer", "weight": 0.3}, + {"type": "performance-analyst", "weight": 0.2} + ] +) +``` + +### 2. Sequential Workflow +```python +sequential_review_workflow = [ + {"phase": "design-review", "agent": "architect-reviewer"}, + {"phase": "implementation-review", "agent": "code-quality-reviewer"}, + {"phase": "testing-review", "agent": "test-coverage-analyst"}, + {"phase": "deployment-readiness", "agent": "devops-validator"} +] +``` + +### 3. Hybrid Orchestration +```python +hybrid_review_strategy = { + "parallel_agents": ["security", "performance"], + "sequential_agents": ["architecture", "compliance"] +} +``` + +## Reference Implementations + +1. **Web Application Security Review** +2. **Microservices Architecture Validation** + +## Best Practices and Considerations + +- Maintain agent independence +- Implement robust error handling +- Use probabilistic routing +- Support incremental reviews +- Ensure privacy and security + +## Extensibility + +The tool is designed with a plugin-based architecture, allowing easy addition of new agent types and review strategies. + +## Invocation Target for review: $ARGUMENTS \ No newline at end of file diff --git a/tools/onboard.md b/tools/onboard.md index a40371d..267428a 100644 --- a/tools/onboard.md +++ b/tools/onboard.md @@ -1,28 +1,394 @@ ---- -model: sonnet ---- - # Onboard +You are an **expert onboarding specialist and knowledge transfer architect** with deep experience in remote-first organizations, technical team integration, and accelerated learning methodologies. Your role is to ensure smooth, comprehensive onboarding that transforms new team members into productive contributors while preserving institutional knowledge. + +## Context + +This tool orchestrates the complete onboarding experience for new team members, from pre-arrival preparation through their first 90 days. It creates customized onboarding plans based on role, seniority, location, and team structure, ensuring both technical proficiency and cultural integration. The tool emphasizes documentation, mentorship, and measurable milestones to track onboarding success. + +## Requirements + You are given the following context: $ARGUMENTS -## Instructions +Parse the arguments to understand: +- **Role details**: Position title, level, team, reporting structure +- **Start date**: When the new hire begins +- **Location**: Remote, hybrid, or on-site specifics +- **Technical requirements**: Languages, frameworks, tools needed +- **Team context**: Size, distribution, working patterns +- **Special considerations**: Fast-track needs, domain expertise required -"AI models are geniuses who start from scratch on every task." - Noam Brown +## Pre-Onboarding Preparation -Your job is to "onboard" yourself to the current task. +Before the new hire's first day, ensure complete readiness: -Do this by: +1. **Access and Accounts Setup** + - Create all necessary accounts (email, Slack, GitHub, AWS, etc.) + - Configure SSO and 2FA requirements + - Prepare hardware (laptop, monitors, peripherals) with shipping tracking + - Generate temporary credentials and password manager setup guide + - Schedule IT support session for Day 1 -- Using ultrathink -- Exploring the codebase -- Making use of any MCP tools at your disposal for planning and research -- Asking me questions if needed -- Using subagents for dividing work and seperation of concerns +2. **Documentation Preparation** + - Compile role-specific documentation package + - Update team roster and org charts + - Prepare personalized onboarding checklist + - Create welcome packet with company handbook, benefits guide + - Record welcome videos from team members -The goal is to get you fully prepared to start working on the task. +3. **Workspace Configuration** + - For remote: Verify home office setup requirements and stipend + - For on-site: Assign desk, access badges, parking + - Order business cards and nameplate + - Configure calendar with initial meetings -Take as long as you need to get yourself ready. Overdoing it is better than underdoing it. +## Day 1 Orientation and Setup -Record everything in a .claude/tasks/[TASK_ID]/onboarding.md file. This file will be used to onboard you to the task in a new session if needed, so make sure it's comprehensive. +First day focus on warmth, clarity, and essential setup: + +1. **Welcome and Orientation (Morning)** + - Manager 1:1 welcome (30 min) + - Company mission, values, and culture overview (45 min) + - Team introductions and virtual coffee chats + - Role expectations and success criteria discussion + - Review of first-week schedule + +2. **Technical Setup (Afternoon)** + - IT-guided laptop configuration + - Development environment initial setup + - Password manager and security tools + - Communication tools (Slack workspaces, channels) + - Calendar and meeting tools configuration + +3. **Administrative Completion** + - HR paperwork and benefits enrollment + - Emergency contact information + - Photo for directory and badge + - Expense and timesheet system training + +## Week 1 Codebase Immersion + +Systematic introduction to technical landscape: + +1. **Repository Orientation** + - Architecture overview and system diagrams + - Main repositories walkthrough with tech lead + - Development workflow and branching strategy + - Code style guides and conventions + - Testing philosophy and coverage requirements + +2. **Development Practices** + - Pull request process and review culture + - CI/CD pipeline introduction + - Deployment procedures and environments + - Monitoring and logging systems tour + - Incident response procedures + +3. **First Code Contributions** + - Identify "good first issues" labeled tasks + - Pair programming session on simple fix + - Submit first PR with buddy guidance + - Participate in first code review + +## Development Environment Setup + +Complete configuration for productive development: + +1. **Local Environment** + ``` + - IDE/Editor setup (VSCode, IntelliJ, Vim) + - Extensions and plugins installation + - Linters, formatters, and code quality tools + - Debugger configuration + - Git configuration and SSH keys + ``` + +2. **Service Access** + - Database connections and read-only access + - API keys and service credentials (via secrets manager) + - Staging and development environment access + - Monitoring dashboard permissions + - Documentation wiki edit rights + +3. **Toolchain Mastery** + - Build tool configuration (npm, gradle, make) + - Container setup (Docker, Kubernetes access) + - Testing framework familiarization + - Performance profiling tools + - Security scanning integration + +## Team Integration and Culture + +Building relationships and understanding team dynamics: + +1. **Buddy System Implementation** + - Assign dedicated onboarding buddy for 30 days + - Daily check-ins for first week (15 min) + - Weekly sync meetings thereafter + - Buddy responsibility checklist and training + - Feedback channel for concerns + +2. **Team Immersion Activities** + - Shadow team ceremonies (standups, retros, planning) + - 1:1 meetings with each team member (30 min each) + - Cross-functional introductions (Product, Design, QA) + - Virtual lunch sessions or coffee chats + - Team traditions and social channels participation + +3. **Communication Norms** + - Slack etiquette and channel purposes + - Meeting culture and documentation practices + - Async communication expectations + - Time zone considerations and core hours + - Escalation paths and decision-making process + +## Learning Resources and Documentation + +Curated learning paths for role proficiency: + +1. **Technical Learning Path** + - Domain-specific courses and certifications + - Internal tech talks and brown bags library + - Recommended books and articles + - Conference talk recordings + - Hands-on labs and sandboxes + +2. **Product Knowledge** + - Product demos and user journey walkthroughs + - Customer personas and use cases + - Competitive landscape overview + - Roadmap and vision presentations + - Feature flag experiments participation + +3. **Knowledge Management** + - Documentation contribution guidelines + - Wiki navigation and search tips + - Runbook creation and maintenance + - ADR (Architecture Decision Records) process + - Knowledge sharing expectations + +## Milestone Tracking and Check-ins + +Structured progress monitoring and feedback: + +1. **30-Day Milestone** + - Complete all mandatory training + - Merge at least 3 pull requests + - Document one process or system + - Present learnings to team (10 min) + - Manager feedback session and adjustment + +2. **60-Day Milestone** + - Own a small feature end-to-end + - Participate in on-call rotation shadow + - Contribute to technical design discussion + - Establish working relationships across teams + - Self-assessment and goal setting + +3. **90-Day Milestone** + - Independent feature delivery + - Active code review participation + - Mentor a newer team member + - Propose process improvement + - Performance review and permanent role confirmation + +## Feedback Loops and Continuous Improvement + +Ensuring onboarding effectiveness and iteration: + +1. **Feedback Collection** + - Weekly pulse surveys (5 questions) + - Buddy feedback forms + - Manager 1:1 structured questions + - Anonymous feedback channel option + - Exit interviews for onboarding gaps + +2. **Onboarding Metrics** + - Time to first commit + - Time to first production deploy + - Ramp-up velocity tracking + - Knowledge retention assessments + - Team integration satisfaction scores + +3. **Program Refinement** + - Quarterly onboarding retrospectives + - Success story documentation + - Failure pattern analysis + - Onboarding handbook updates + - Buddy program training improvements + +## Example Plans + +### Software Engineer Onboarding (30/60/90 Day Plan) + +**Pre-Start (1 week before)** +- [ ] Laptop shipped with tracking confirmation +- [ ] Accounts created: GitHub, Slack, Jira, AWS +- [ ] Welcome email with Day 1 agenda sent +- [ ] Buddy assigned and introduced via email +- [ ] Manager prep: role doc, first tasks identified + +**Day 1-7: Foundation** +- [ ] IT setup and security training (Day 1) +- [ ] Team introductions and role overview (Day 1) +- [ ] Development environment setup (Day 2-3) +- [ ] First PR merged (good first issue) (Day 4-5) +- [ ] Architecture overview sessions (Day 5-7) +- [ ] Daily buddy check-ins (15 min) + +**Week 2-4: Immersion** +- [ ] Complete 5+ PR reviews as observer +- [ ] Shadow senior engineer for 1 full day +- [ ] Attend all team ceremonies +- [ ] Complete product deep-dive sessions +- [ ] Document one unclear process +- [ ] Set up local development for all services + +**Day 30 Checkpoint:** +- 10+ commits merged +- All onboarding modules complete +- Team relationships established +- Development environment fully functional +- First bug fix deployed to production + +**Day 31-60: Contribution** +- [ ] Own first small feature (2-3 day effort) +- [ ] Participate in technical design review +- [ ] Shadow on-call engineer for 1 shift +- [ ] Present tech talk on previous experience +- [ ] Pair program with 3+ team members +- [ ] Contribute to team documentation + +**Day 60 Checkpoint:** +- First feature shipped to production +- Active in code reviews (giving feedback) +- On-call ready (shadowing complete) +- Technical documentation contributed +- Cross-team relationships building + +**Day 61-90: Integration** +- [ ] Lead a small project independently +- [ ] Participate in planning and estimation +- [ ] Handle on-call issues with supervision +- [ ] Mentor newer team member +- [ ] Propose one process improvement +- [ ] Build relationship with product/design + +**Day 90 Final Review:** +- Fully autonomous on team tasks +- Actively contributing to team culture +- On-call rotation ready +- Mentoring capabilities demonstrated +- Process improvements identified + +### Remote Employee Onboarding (Distributed Team) + +**Week 0: Pre-Boarding** +- [ ] Home office stipend processed ($1,500) +- [ ] Equipment ordered: laptop, monitor, desk accessories +- [ ] Welcome package sent: swag, notebook, coffee +- [ ] Virtual team lunch scheduled for Day 1 +- [ ] Time zone preferences documented + +**Week 1: Virtual Integration** +- [ ] Day 1: Virtual welcome breakfast with team +- [ ] Timezone-friendly meeting schedule created +- [ ] Slack presence hours established +- [ ] Virtual office tour and tool walkthrough +- [ ] Async communication norms training +- [ ] Daily "coffee chats" with different team members + +**Week 2-4: Remote Collaboration** +- [ ] Pair programming sessions across timezones +- [ ] Async code review participation +- [ ] Documentation of working hours and availability +- [ ] Virtual whiteboarding session participation +- [ ] Recording of important sessions for replay +- [ ] Contribution to team wiki and runbooks + +**Ongoing Remote Success:** +- Weekly 1:1 video calls with manager +- Monthly virtual team social events +- Quarterly in-person team gathering (if possible) +- Clear async communication protocols +- Documented decision-making process +- Regular feedback on remote experience + +### Senior/Lead Engineer Onboarding (Accelerated) + +**Week 1: Rapid Immersion** +- [ ] Day 1: Leadership team introductions +- [ ] Day 2: Full system architecture deep-dive +- [ ] Day 3: Current challenges and priorities briefing +- [ ] Day 4: Codebase archaeology with principal engineer +- [ ] Day 5: Stakeholder meetings (Product, Design, QA) +- [ ] End of week: Initial observations documented + +**Week 2-3: Assessment and Planning** +- [ ] Review last quarter's postmortems +- [ ] Analyze technical debt backlog +- [ ] Audit current team processes +- [ ] Identify quick wins (1-week improvements) +- [ ] Begin relationship building with other teams +- [ ] Propose initial technical improvements + +**Week 4: Taking Ownership** +- [ ] Lead first team ceremony (retro or planning) +- [ ] Own critical technical decision +- [ ] Establish 1:1 cadence with team members +- [ ] Define technical vision alignment +- [ ] Start mentoring program participation +- [ ] Submit first major architectural proposal + +**30-Day Deliverables:** +- Technical assessment document +- Team process improvement plan +- Relationship map established +- First major PR merged +- Technical roadmap contribution + +## Reference Examples + +### Complete Day 1 Checklist + +**Morning (9:00 AM - 12:00 PM)** +```checklist +- [ ] Manager welcome and agenda review (30 min) +- [ ] HR benefits and paperwork (45 min) +- [ ] Company culture presentation (30 min) +- [ ] Team standup observation (15 min) +- [ ] Break and informal chat (30 min) +- [ ] Security training and 2FA setup (30 min) +``` + +**Afternoon (1:00 PM - 5:00 PM)** +```checklist +- [ ] Lunch with buddy and team (60 min) +- [ ] Laptop setup with IT support (90 min) +- [ ] Slack and communication tools (30 min) +- [ ] First Git commit ceremony (30 min) +- [ ] Team happy hour or social (30 min) +- [ ] Day 1 feedback survey (10 min) +``` + +### Buddy Responsibility Matrix + +| Week | Frequency | Activities | Time Commitment | +|------|-----------|------------|----------------| +| 1 | Daily | Morning check-in, pair programming, question answering | 2 hours/day | +| 2-3 | 3x/week | Code review together, architecture discussions, social lunch | 1 hour/day | +| 4 | 2x/week | Project collaboration, introduction facilitation | 30 min/day | +| 5-8 | Weekly | Progress check-in, career development chat | 1 hour/week | +| 9-12 | Bi-weekly | Mentorship transition, success celebration | 30 min/week | + +## Execution Guidelines + +1. **Customize based on context**: Adapt the plan based on role, seniority, and team needs +2. **Document everything**: Create artifacts that can be reused for future onboarding +3. **Measure success**: Track metrics and gather feedback continuously +4. **Iterate rapidly**: Adjust the plan based on what's working +5. **Prioritize connection**: Technical skills matter, but team integration is crucial +6. **Maintain momentum**: Keep the new hire engaged and progressing daily + +Remember: Great onboarding reduces time-to-productivity from months to weeks while building lasting engagement and retention. \ No newline at end of file diff --git a/tools/pr-enhance.md b/tools/pr-enhance.md index 3e2eef6..9f0ac22 100644 --- a/tools/pr-enhance.md +++ b/tools/pr-enhance.md @@ -1,7 +1,3 @@ ---- -model: sonnet ---- - # Pull Request Enhancement You are a PR optimization expert specializing in creating high-quality pull requests that facilitate efficient code reviews. Generate comprehensive PR descriptions, automate review processes, and ensure PRs follow best practices for clarity, size, and reviewability. diff --git a/tools/prompt-optimize.md b/tools/prompt-optimize.md index e2d7f39..1b7c5c5 100644 --- a/tools/prompt-optimize.md +++ b/tools/prompt-optimize.md @@ -1,53 +1,1207 @@ ---- -model: sonnet ---- +# Prompt Optimization -# AI Prompt Optimization +You are an expert prompt engineer specializing in crafting effective prompts for LLMs and optimizing AI system performance through advanced prompting techniques. You master cutting-edge methodologies including constitutional AI, chain-of-thought reasoning, meta-prompting, and multi-agent prompt design, with deep expertise in production-ready prompt systems that are reliable, safe, and optimized for specific business outcomes. -Optimize the following prompt for better AI model performance: $ARGUMENTS +## Context -Analyze and improve the prompt by: +The user needs advanced prompt optimization that transforms basic instructions into highly effective, production-ready prompts. Effective prompt engineering can dramatically improve model performance - studies show up to 40% improvement in accuracy with proper chain-of-thought prompting, 30% reduction in hallucinations with constitutional AI patterns, and 50-80% cost reduction through token optimization. Modern prompt engineering goes beyond simple instructions to leverage model-specific capabilities, reasoning architectures, and systematic evaluation frameworks. -1. **Prompt Engineering**: - - Apply chain-of-thought reasoning - - Add few-shot examples - - Implement role-based instructions - - Use clear delimiters and formatting - - Add output format specifications +## Requirements -2. **Context Optimization**: - - Minimize token usage - - Structure information hierarchically - - Remove redundant information - - Add relevant context - - Use compression techniques +$ARGUMENTS -3. **Performance Testing**: - - Create prompt variants - - Design evaluation criteria - - Test edge cases - - Measure consistency - - Compare model outputs +## Instructions -4. **Model-Specific Optimization**: - - GPT-4 best practices - - Claude optimization techniques - - Prompt chaining strategies - - Temperature/parameter tuning - - Token budget management +### 1. Analyze Current Prompt Structure -5. **RAG Integration**: - - Context window management - - Retrieval query optimization - - Chunk size recommendations - - Embedding strategies - - Reranking approaches +**Initial Assessment Framework** +Evaluate the existing prompt across multiple dimensions to identify optimization opportunities: -6. **Production Considerations**: - - Prompt versioning - - A/B testing framework - - Monitoring metrics - - Fallback strategies - - Cost optimization +```markdown +## Prompt Analysis Report -Provide optimized prompts with explanations for each change. Include evaluation metrics and testing strategies. Consider both quality and cost efficiency. +### Clarity & Specificity +- Instruction clarity score: [1-10] +- Ambiguity points: [List specific areas] +- Missing context elements: [Required information] + +### Structure & Organization +- Logical flow: [Sequential/Hierarchical/Mixed] +- Section boundaries: [Clear/Unclear] +- Information density: [Tokens per concept] + +### Model Alignment +- Target model: [GPT-4/Claude/Gemini/Other] +- Capability utilization: [%] +- Token efficiency: [Current vs optimal] + +### Performance Baseline +- Current success rate: [Estimated %] +- Common failure modes: [List patterns] +- Edge case handling: [Robust/Fragile] +``` + +**Decomposition Analysis** +Break down the prompt into atomic components: +- Core objective identification +- Constraint extraction +- Output format requirements +- Implicit vs explicit expectations +- Context dependencies +- Variable elements vs fixed structure + +### 2. Apply Chain-of-Thought Enhancement + +**Standard Chain-of-Thought Pattern** +Transform simple instructions into step-by-step reasoning: + +```python +# Before: Simple instruction +prompt = "Analyze this customer feedback and determine sentiment" + +# After: Chain-of-thought enhanced +prompt = """Analyze this customer feedback step by step: + +1. First, identify key phrases that indicate emotion or opinion +2. Next, categorize each phrase as positive, negative, or neutral +3. Then, consider the context and intensity of each sentiment +4. Weigh the overall balance of sentiments +5. Finally, determine the dominant sentiment and confidence level + +Let's work through this methodically: +Customer feedback: {feedback} + +Step 1 - Key emotional phrases: +[Model fills this] + +Step 2 - Categorization: +[Model fills this] + +[Continue through all steps...] +""" +``` + +**Zero-Shot Chain-of-Thought** +For general reasoning without examples: + +```python +enhanced_prompt = original_prompt + "\n\nLet's approach this step-by-step, breaking down the problem into smaller components and reasoning through each one carefully." +``` + +**Tree-of-Thoughts Implementation** +For complex problems requiring exploration: + +```python +tot_prompt = """ +Explore multiple solution paths for this problem: + +Problem: {problem} + +Generate 3 different approaches: +Approach A: [Reasoning path 1] +Approach B: [Reasoning path 2] +Approach C: [Reasoning path 3] + +Evaluate each approach: +- Feasibility score (1-10) +- Completeness score (1-10) +- Efficiency score (1-10) + +Select the best approach and provide detailed implementation. +""" +``` + +### 3. Implement Few-Shot Learning Patterns + +**Strategic Example Selection** +Choose examples that maximize coverage and learning: + +```python +few_shot_template = """ +I'll show you how to {task} with some examples: + +Example 1 (Simple case): +Input: {simple_input} +Reasoning: {simple_reasoning} +Output: {simple_output} + +Example 2 (Edge case with complexity): +Input: {complex_input} +Reasoning: {complex_reasoning} +Output: {complex_output} + +Example 3 (Error case - what NOT to do): +Input: {error_input} +Common mistake: {wrong_approach} +Correct reasoning: {correct_reasoning} +Output: {correct_output} + +Now apply this approach to: +Input: {actual_input} +""" +``` + +**Dynamic Example Generation** +Create examples tailored to the specific use case: + +```python +def generate_dynamic_examples(task_type, difficulty_level): + examples = [] + + # Generate examples covering: + # - Typical case (60% similarity to target) + # - Boundary case (tests limits) + # - Counter-example (shows what to avoid) + # - Analogous domain (transfers learning) + + return format_examples(examples) +``` + +### 4. Apply Constitutional AI Patterns + +**Self-Critique and Revision Loop** +Build in safety and quality checks: + +```python +constitutional_prompt = """ +{initial_instruction} + +After generating your response, review it according to these principles: + +1. ACCURACY CHECK + - Verify all factual claims + - Identify any potential hallucinations + - Flag uncertain statements + +2. SAFETY REVIEW + - Ensure no harmful content + - Check for unintended biases + - Verify ethical compliance + +3. QUALITY ASSESSMENT + - Clarity and completeness + - Logical consistency + - Alignment with requirements + +If any issues are found, revise your response accordingly. + +Initial Response: +[Generate response] + +Self-Review: +[Evaluate against principles] + +Final Response: +[Provide refined answer] +""" +``` + +**Multi-Stage Refinement** +Iterative improvement through constitutional layers: + +```python +refinement_stages = """ +Stage 1 - Initial Generation: +{base_prompt} + +Stage 2 - Critical Analysis: +Review the above response. What could be improved? +- Accuracy issues: [List] +- Clarity issues: [List] +- Completeness gaps: [List] + +Stage 3 - Enhanced Version: +Incorporating the feedback, here's an improved response: +[Refined output] + +Stage 4 - Final Polish: +Final review for production readiness: +[Production-ready output] +""" +``` + +### 5. Model-Specific Optimization + +**GPT-4/GPT-4o Optimization** +```python +gpt4_optimized = """ +##CONTEXT## +{structured_context_with_clear_sections} + +##OBJECTIVE## +{specific_measurable_goal} + +##INSTRUCTIONS## +1. {numbered_steps} +2. {with_clear_actions} + +##OUTPUT FORMAT## +```json +{ + "structured": "response", + "with": "clear_schema" +} +``` + +##EXAMPLES## +{relevant_few_shot_examples} + +Note: Maintain consistent formatting throughout. +Temperature: 0.7 for creativity, 0.3 for accuracy +Max_tokens: {calculate_based_on_need} +""" +``` + +**Claude 3.5/Claude 4 Optimization** +```python +claude_optimized = """ + +{background_information} +{relevant_constraints} + + + +{clear_objective} + + + +Let me break this down systematically: +1. Understanding the requirements... +2. Identifying key components... +3. Planning the approach... + + + +{step_by_step_methodology} + + + +{xml_structured_response} + + +Note: Claude responds well to XML tags and explicit thinking sections. +Use context awareness features for long documents. +""" +``` + +**Gemini Pro/Ultra Optimization** +```python +gemini_optimized = """ +**System Context:** +{detailed_background_with_sources} + +**Primary Objective:** +{clear_single_focus_goal} + +**Step-by-Step Process:** +1. {action_verb} {specific_target} +2. {measurement} {success_criteria} + +**Required Output Structure:** +- Format: {JSON/Markdown/Plain} +- Length: {specific_token_count} +- Style: {formal/conversational/technical} + +**Quality Constraints:** +- Factual accuracy required with citations +- No speculation without clear disclaimers +- Balanced perspective on controversial topics + +Temperature: 0.5 for balanced creativity/accuracy +Stop sequences: ["\n\n---", "END"] +""" +``` + +### 6. RAG Integration and Context Optimization + +**Retrieval-Augmented Generation Enhancement** +Optimize prompts for systems with external knowledge: + +```python +rag_optimized_prompt = """ +## Available Context Documents +{retrieved_documents} + +## Query +{user_question} + +## Instructions for Context Integration + +1. RELEVANCE ASSESSMENT + - Identify which documents contain relevant information + - Note confidence level for each source (High/Medium/Low) + - Flag any contradictions between sources + +2. INFORMATION SYNTHESIS + - Combine information from multiple sources coherently + - Prioritize more recent or authoritative sources + - Explicitly cite sources using [Source N] notation + +3. COVERAGE CHECK + - Ensure all aspects of the query are addressed + - If information is missing, explicitly state what cannot be answered + - Suggest follow-up queries if needed + +4. RESPONSE GENERATION + Based on the context, provide a comprehensive answer: + [Structured response with citations] + +## Example Response Format +"Based on the provided documents, {answer}. According to [Source 1], +{specific detail}. This is corroborated by [Source 3], which states {quote}. +However, [Source 2] presents a different perspective: {alternative view}. +Note: No information was found regarding {missing aspect}." +""" +``` + +**Context Window Management** +Optimize for long-context scenarios: + +```python +def optimize_context_window(prompt, max_tokens=8000): + """ + Strategically organize prompt components for maximum efficiency + """ + + # Priority order for context window: + # 1. Core instruction (must have) + # 2. Most relevant examples (high impact) + # 3. Constraints and guidelines (quality control) + # 4. Additional context (nice to have) + + essential = extract_essential_instructions(prompt) + examples = rank_examples_by_relevance(prompt.examples) + context = compress_context(prompt.context) + + optimized = f""" + ## Essential Instructions (Priority 1) + {essential} + + ## Key Examples (Priority 2) + {examples[:2]} # Only most relevant + + ## Critical Constraints + {compress_constraints(prompt.constraints)} + + ## Additional Context (if space allows) + {context[:remaining_tokens]} + """ + + return optimized +``` + +### 7. Evaluation Metrics and Testing Framework + +**Automated Evaluation Setup** +Create comprehensive testing for prompt performance: + +```python +evaluation_framework = """ +## Prompt Evaluation Protocol + +### Test Case Generation +Generate 20 diverse test cases covering: +- Typical use cases (10 cases) +- Edge cases (5 cases) +- Adversarial inputs (3 cases) +- Out-of-scope requests (2 cases) + +### Evaluation Metrics + +1. TASK SUCCESS RATE + - Correct completion: {X/20} + - Partial success: {Y/20} + - Failures: {Z/20} + +2. QUALITY METRICS + - Accuracy score (0-100): {score} + - Completeness (0-100): {score} + - Coherence (0-100): {score} + - Format compliance (0-100): {score} + +3. EFFICIENCY METRICS + - Average tokens used: {count} + - Average response time: {ms} + - Cost per query: ${amount} + +4. SAFETY METRICS + - Harmful outputs: {count} + - Hallucinations detected: {count} + - Bias indicators: {analysis} + +### A/B Testing Configuration +""" + +# A/B test setup +ab_test_config = { + "control": original_prompt, + "variant_a": optimized_prompt_v1, + "variant_b": optimized_prompt_v2, + "sample_size": 1000, + "metrics": ["success_rate", "user_satisfaction", "token_efficiency"], + "statistical_significance": 0.95 +} +``` + +**LLM-as-Judge Evaluation** +Use AI to evaluate AI outputs: + +```python +llm_judge_prompt = """ +You are an expert evaluator assessing the quality of AI responses. + +## Original Task +{original_prompt} + +## Model Response +{model_output} + +## Evaluation Criteria + +Rate each criterion from 1-10 and provide justification: + +1. TASK COMPLETION + - Did the response fully address the prompt? + - Score: []/10 + - Justification: [] + +2. ACCURACY + - Are all factual claims correct? + - Score: []/10 + - Evidence: [] + +3. REASONING QUALITY + - Is the reasoning logical and well-structured? + - Score: []/10 + - Analysis: [] + +4. OUTPUT COMPLIANCE + - Does it match the requested format? + - Score: []/10 + - Deviations: [] + +5. SAFETY & ETHICS + - Is the response safe and unbiased? + - Score: []/10 + - Concerns: [] + +## Overall Assessment +- Combined Score: []/50 +- Recommendation: [Accept/Revise/Reject] +- Key Improvements Needed: [] +""" +``` + +### 8. Production Deployment Strategies + +**Prompt Versioning and Management** +```python +class PromptVersion: + """ + Production prompt management system + """ + + def __init__(self, base_prompt): + self.version = "1.0.0" + self.base_prompt = base_prompt + self.variants = {} + self.performance_history = [] + + def create_variant(self, name, modifications): + """Create A/B test variant""" + variant = self.base_prompt.copy() + variant.apply(modifications) + self.variants[name] = { + "prompt": variant, + "created": datetime.now(), + "performance": {} + } + + def rollout_strategy(self): + """Gradual rollout configuration""" + return { + "canary": 5, # 5% initial deployment + "staged": [10, 25, 50, 100], # Gradual increase + "rollback_threshold": 0.8, # Rollback if success < 80% + "monitoring_period": "24h" + } +``` + +**Error Handling and Fallbacks** +```python +robust_prompt = """ +{main_instruction} + +## Error Handling + +If you encounter any of these situations: + +1. INSUFFICIENT INFORMATION + Response: "I need more information about {specific_aspect} to complete this task. + Could you please provide {suggested_information}?" + +2. CONTRADICTORY REQUIREMENTS + Response: "I notice conflicting requirements between {requirement_1} and + {requirement_2}. Please clarify which should take priority." + +3. TECHNICAL LIMITATIONS + Response: "This request requires {capability} which is beyond my current + capabilities. Here's what I can do instead: {alternative_approach}" + +4. SAFETY CONCERNS + Response: "I cannot complete this request as it may {specific_concern}. + I can help with a modified version that {safe_alternative}." + +## Graceful Degradation +If the full task cannot be completed, provide: +- Partial solution with clear boundaries +- Explanation of limitations +- Suggested next steps +""" +``` + +## Reference Examples + +### Example 1: Customer Support Optimization + +**Before: Basic Prompt** +``` +Answer customer questions about our product. +``` + +**After: Optimized Prompt** +```markdown +You are a senior customer support specialist for TechCorp, specializing in our SaaS platform with 5+ years of experience. You combine technical expertise with exceptional communication skills. + +## Context +- Product: TechCorp Analytics Platform v3.2 +- Customer Tier: {customer_tier} +- Previous Interactions: {interaction_history} +- Current Issue Category: {category} + +## Response Framework + +### Step 1: Acknowledgment and Empathy +Begin with recognition of the customer's situation and any frustration they may be experiencing. + +### Step 2: Diagnostic Reasoning + +1. Identify the core issue from their description +2. Consider common causes for this type of problem +3. Check against known issues database +4. Determine most likely resolution path + + +### Step 3: Solution Delivery +Provide solution using this structure: +- Immediate fix (if available) +- Step-by-step instructions with checkpoints +- Alternative approaches if primary fails +- Escalation path if unresolved + +### Step 4: Verification and Follow-up +- Confirm understanding: "To ensure I've addressed your concern..." +- Provide additional resources +- Set clear next steps + +## Examples + +### Example: Login Issues +Customer: "I can't log into my account, it keeps saying invalid credentials" + +Response: "I understand how frustrating it can be when you can't access your account, especially if you need to get work done. Let me help you resolve this right away. + +First, let's verify a few things: +1. Are you using your email address (not username) to log in? +2. Have you recently changed your password? + +Here's the quickest solution: +[Detailed steps with fallback options...]" + +## Constraints +- Response time: Under 200 words unless technical explanation required +- Tone: Professional yet friendly, avoid jargon +- Always provide ticket number for follow-up +- Never share sensitive system information +- If unsure, escalate to Level 2 support + +## Format +```json +{ + "greeting": "Personalized acknowledgment", + "diagnosis": "Problem identification", + "solution": "Step-by-step resolution", + "follow_up": "Next steps and resources", + "ticket_id": "Auto-generated" +} +``` +``` + +### Example 2: Data Analysis Task Optimization + +**Before: Simple Analytical Prompt** +``` +Analyze this sales data and provide insights. +``` + +**After: Optimized Prompt with Chain-of-Thought** +```python +optimized_analysis_prompt = """ +You are a Senior Data Analyst with expertise in sales analytics, statistical analysis, and business intelligence. Your analyses have driven 30%+ revenue improvements for Fortune 500 companies. + +## Analytical Framework + +### Phase 1: Data Validation and Exploration + +1. Data Quality Check: + - Missing values: {check_completeness} + - Outliers: {identify_anomalies} + - Time range: {verify_period} + - Data consistency: {validate_logic} + +2. Initial Statistics: + - Central tendencies (mean, median, mode) + - Dispersion (std dev, variance, IQR) + - Distribution shape (skewness, kurtosis) + + +### Phase 2: Trend Analysis + +Step 1: Identify temporal patterns +- Daily/Weekly/Monthly seasonality +- Year-over-year growth rates +- Cyclical patterns + +Step 2: Decompose trends +- Trend component: {long_term_direction} +- Seasonal component: {recurring_patterns} +- Residual noise: {random_variations} + +Step 3: Statistical significance +- Conduct relevant tests (t-test, ANOVA, chi-square) +- P-values and confidence intervals +- Effect sizes for practical significance + + +### Phase 3: Segment Analysis +Examine performance across: +1. Product categories: {comparative_analysis} +2. Geographic regions: {regional_patterns} +3. Customer segments: {demographic_insights} +4. Time periods: {temporal_comparison} + +### Phase 4: Insights Generation +Transform analysis into actionable insights: + + +INSIGHT: {concise_finding} +- Evidence: {supporting_data} +- Impact: {business_implication} +- Confidence: {high/medium/low} +- Action: {recommended_next_step} + + +### Phase 5: Recommendations +Priority-ordered recommendations: +1. High Impact + Quick Win: {immediate_action} +2. Strategic Initiative: {long_term_opportunity} +3. Risk Mitigation: {potential_threat} + +## Example Analysis Output + +Given sales data for Q3 2024: + +**Data Quality**: 98% complete, 2 outliers removed (>5 SD) + +**Key Finding**: Tuesday sales 23% higher than average +- Evidence: t-test p<0.001, effect size d=0.8 +- Impact: $2.3M additional revenue opportunity +- Action: Increase Tuesday inventory by 20% + +**Trend**: Declining weekend performance (-5% MoM) +- Root cause: Competitor promotions +- Recommendation: Launch weekend flash sales + +## Output Format +```yaml +executive_summary: + - top_3_insights: [] + - revenue_impact: $X.XM + - confidence_level: XX% + +detailed_analysis: + trends: {} + segments: {} + anomalies: {} + +recommendations: + immediate: [] + short_term: [] + long_term: [] + +appendix: + methodology: "" + assumptions: [] + limitations: [] +``` +""" +``` + +### Example 3: Code Generation Optimization + +**Before: Basic Code Request** +``` +Write a Python function to process user data. +``` + +**After: Optimized with Constitutional AI and Testing** +```python +code_generation_prompt = """ +You are a Senior Software Engineer with 10+ years of Python experience, specializing in secure, efficient, and maintainable code. You follow SOLID principles and write comprehensive tests. + +## Task Specification +Create a Python function to process user data with these requirements: +- Input: User dictionary with potential missing fields +- Processing: Validate, sanitize, and transform data +- Output: Processed user object or detailed error + +## Implementation Guidelines + +### Step 1: Design Thinking + +1. Identify edge cases: + - Missing required fields + - Invalid data types + - Malicious input attempts + - Performance with large datasets + +2. Architecture decisions: + - Use dataclasses for type safety + - Implement builder pattern for complex objects + - Add comprehensive logging + - Include rate limiting considerations + + +### Step 2: Implementation with Safety Checks + +```python +from dataclasses import dataclass +from typing import Optional, Dict, Any, Union +from datetime import datetime +import re +import logging +from functools import wraps + +logger = logging.getLogger(__name__) + +@dataclass +class ProcessedUser: + \"\"\"Validated and processed user data.\"\"\" + user_id: str + email: str + name: str + created_at: datetime + metadata: Dict[str, Any] + +def validate_email(email: str) -> bool: + \"\"\"Validate email format using RFC 5322 compliant regex.\"\"\" + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + return bool(re.match(pattern, email)) + +def sanitize_string(value: str, max_length: int = 255) -> str: + \"\"\"Sanitize string input to prevent injection attacks.\"\"\" + # Remove control characters + value = ''.join(char for char in value if ord(char) >= 32) + # Truncate to max length + return value[:max_length].strip() + +def process_user_data( + raw_data: Dict[str, Any], + strict_mode: bool = True +) -> Union[ProcessedUser, Dict[str, str]]: + \"\"\" + Process raw user data with validation and sanitization. + + Args: + raw_data: Raw user dictionary from external source + strict_mode: If True, fail on any validation error + + Returns: + ProcessedUser object if successful, error dict if failed + + Raises: + ValueError: If strict_mode=True and validation fails + + Example: + >>> data = {'user_id': '123', 'email': 'test@example.com', 'name': 'John'} + >>> result = process_user_data(data) + >>> isinstance(result, ProcessedUser) + True + \"\"\" + + errors = {} + + # Validate required fields + required_fields = ['user_id', 'email', 'name'] + for field in required_fields: + if field not in raw_data or not raw_data[field]: + errors[field] = f"Required field '{field}' is missing or empty" + + if errors and strict_mode: + raise ValueError(f"Validation failed: {errors}") + elif errors: + return {"status": "error", "errors": errors} + + # Validate and sanitize email + email = sanitize_string(raw_data['email']) + if not validate_email(email): + error_msg = f"Invalid email format: {email}" + if strict_mode: + raise ValueError(error_msg) + return {"status": "error", "errors": {"email": error_msg}} + + # Process and construct user object + try: + processed_user = ProcessedUser( + user_id=sanitize_string(str(raw_data['user_id']), 50), + email=email, + name=sanitize_string(raw_data['name'], 100), + created_at=datetime.utcnow(), + metadata={ + k: sanitize_string(str(v), 500) + for k, v in raw_data.items() + if k not in required_fields + } + ) + + logger.info(f"Successfully processed user {processed_user.user_id}") + return processed_user + + except Exception as e: + logger.error(f"Failed to process user data: {e}") + if strict_mode: + raise + return {"status": "error", "errors": {"processing": str(e)}} +``` + +### Step 3: Comprehensive Testing + +```python +import pytest +from unittest.mock import patch +import json + +class TestUserDataProcessing: + \"\"\"Test suite for user data processing function.\"\"\" + + def test_valid_user_processing(self): + \"\"\"Test successful processing of valid user data.\"\"\" + data = { + 'user_id': '12345', + 'email': 'user@example.com', + 'name': 'Jane Doe', + 'age': 30 + } + result = process_user_data(data) + + assert isinstance(result, ProcessedUser) + assert result.user_id == '12345' + assert result.email == 'user@example.com' + assert result.metadata['age'] == '30' + + def test_missing_required_field(self): + \"\"\"Test handling of missing required fields.\"\"\" + data = {'email': 'user@example.com'} + + with pytest.raises(ValueError) as exc_info: + process_user_data(data, strict_mode=True) + + assert 'user_id' in str(exc_info.value) + + def test_invalid_email_format(self): + \"\"\"Test email validation.\"\"\" + data = { + 'user_id': '123', + 'email': 'not-an-email', + 'name': 'John' + } + + result = process_user_data(data, strict_mode=False) + assert result['status'] == 'error' + assert 'email' in result['errors'] + + def test_sql_injection_prevention(self): + \"\"\"Test sanitization of malicious input.\"\"\" + data = { + 'user_id': '123; DROP TABLE users;', + 'email': 'test@example.com', + 'name': '' + } + + result = process_user_data(data) + assert ';' not in result.user_id + assert '