mirror of
https://github.com/wshobson/agents.git
synced 2026-03-18 09:37:15 +00:00
feat: add 5 new specialized agents with 20 skills
Add domain expert agents with comprehensive skill sets: - service-mesh-expert (cloud-infrastructure): Istio/Linkerd patterns, mTLS, observability - event-sourcing-architect (backend-development): CQRS, event stores, projections, sagas - vector-database-engineer (llm-application-dev): embeddings, similarity search, hybrid search - monorepo-architect (developer-essentials): Nx, Turborepo, Bazel, pnpm workspaces - threat-modeling-expert (security-scanning): STRIDE, attack trees, security requirements Update all documentation to reflect correct counts: - 67 plugins, 99 agents, 107 skills, 71 commands
This commit is contained in:
Seth Hobson
552
plugins/backend-development/skills/cqrs-implementation/SKILL.md
Normal file
552
plugins/backend-development/skills/cqrs-implementation/SKILL.md
Normal file
@@ -0,0 +1,552 @@
|
||||
---
|
||||
name: cqrs-implementation
|
||||
description: Implement Command Query Responsibility Segregation for scalable architectures. Use when separating read and write models, optimizing query performance, or building event-sourced systems.
|
||||
---
|
||||
|
||||
# CQRS Implementation
|
||||
|
||||
Comprehensive guide to implementing CQRS (Command Query Responsibility Segregation) patterns.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Separating read and write concerns
|
||||
- Scaling reads independently from writes
|
||||
- Building event-sourced systems
|
||||
- Optimizing complex query scenarios
|
||||
- Different read/write data models needed
|
||||
- High-performance reporting requirements
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### 1. CQRS Architecture
|
||||
|
||||
```
|
||||
┌─────────────┐
|
||||
│ Client │
|
||||
└──────┬──────┘
|
||||
│
|
||||
┌────────────┴────────────┐
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────┐ ┌─────────────┐
|
||||
│ Commands │ │ Queries │
|
||||
│ API │ │ API │
|
||||
└──────┬──────┘ └──────┬──────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────┐ ┌─────────────┐
|
||||
│ Command │ │ Query │
|
||||
│ Handlers │ │ Handlers │
|
||||
└──────┬──────┘ └──────┬──────┘
|
||||
│ │
|
||||
▼ ▼
|
||||
┌─────────────┐ ┌─────────────┐
|
||||
│ Write │─────────►│ Read │
|
||||
│ Model │ Events │ Model │
|
||||
└─────────────┘ └─────────────┘
|
||||
```
|
||||
|
||||
### 2. Key Components
|
||||
|
||||
| Component | Responsibility |
|
||||
|-----------|---------------|
|
||||
| **Command** | Intent to change state |
|
||||
| **Command Handler** | Validates and executes commands |
|
||||
| **Event** | Record of state change |
|
||||
| **Query** | Request for data |
|
||||
| **Query Handler** | Retrieves data from read model |
|
||||
| **Projector** | Updates read model from events |
|
||||
|
||||
## Templates
|
||||
|
||||
### Template 1: Command Infrastructure
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeVar, Generic, Dict, Any, Type
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
# Command base
|
||||
@dataclass
|
||||
class Command:
|
||||
command_id: str = None
|
||||
timestamp: datetime = None
|
||||
|
||||
def __post_init__(self):
|
||||
self.command_id = self.command_id or str(uuid.uuid4())
|
||||
self.timestamp = self.timestamp or datetime.utcnow()
|
||||
|
||||
|
||||
# Concrete commands
|
||||
@dataclass
|
||||
class CreateOrder(Command):
|
||||
customer_id: str
|
||||
items: list
|
||||
shipping_address: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class AddOrderItem(Command):
|
||||
order_id: str
|
||||
product_id: str
|
||||
quantity: int
|
||||
price: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class CancelOrder(Command):
|
||||
order_id: str
|
||||
reason: str
|
||||
|
||||
|
||||
# Command handler base
|
||||
T = TypeVar('T', bound=Command)
|
||||
|
||||
class CommandHandler(ABC, Generic[T]):
|
||||
@abstractmethod
|
||||
async def handle(self, command: T) -> Any:
|
||||
pass
|
||||
|
||||
|
||||
# Command bus
|
||||
class CommandBus:
|
||||
def __init__(self):
|
||||
self._handlers: Dict[Type[Command], CommandHandler] = {}
|
||||
|
||||
def register(self, command_type: Type[Command], handler: CommandHandler):
|
||||
self._handlers[command_type] = handler
|
||||
|
||||
async def dispatch(self, command: Command) -> Any:
|
||||
handler = self._handlers.get(type(command))
|
||||
if not handler:
|
||||
raise ValueError(f"No handler for {type(command).__name__}")
|
||||
return await handler.handle(command)
|
||||
|
||||
|
||||
# Command handler implementation
|
||||
class CreateOrderHandler(CommandHandler[CreateOrder]):
|
||||
def __init__(self, order_repository, event_store):
|
||||
self.order_repository = order_repository
|
||||
self.event_store = event_store
|
||||
|
||||
async def handle(self, command: CreateOrder) -> str:
|
||||
# Validate
|
||||
if not command.items:
|
||||
raise ValueError("Order must have at least one item")
|
||||
|
||||
# Create aggregate
|
||||
order = Order.create(
|
||||
customer_id=command.customer_id,
|
||||
items=command.items,
|
||||
shipping_address=command.shipping_address
|
||||
)
|
||||
|
||||
# Persist events
|
||||
await self.event_store.append_events(
|
||||
stream_id=f"Order-{order.id}",
|
||||
stream_type="Order",
|
||||
events=order.uncommitted_events
|
||||
)
|
||||
|
||||
return order.id
|
||||
```
|
||||
|
||||
### Template 2: Query Infrastructure
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeVar, Generic, List, Optional
|
||||
|
||||
# Query base
|
||||
@dataclass
|
||||
class Query:
|
||||
pass
|
||||
|
||||
|
||||
# Concrete queries
|
||||
@dataclass
|
||||
class GetOrderById(Query):
|
||||
order_id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class GetCustomerOrders(Query):
|
||||
customer_id: str
|
||||
status: Optional[str] = None
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchOrders(Query):
|
||||
query: str
|
||||
filters: dict = None
|
||||
sort_by: str = "created_at"
|
||||
sort_order: str = "desc"
|
||||
|
||||
|
||||
# Query result types
|
||||
@dataclass
|
||||
class OrderView:
|
||||
order_id: str
|
||||
customer_id: str
|
||||
status: str
|
||||
total_amount: float
|
||||
item_count: int
|
||||
created_at: datetime
|
||||
shipped_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaginatedResult(Generic[T]):
|
||||
items: List[T]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
|
||||
@property
|
||||
def total_pages(self) -> int:
|
||||
return (self.total + self.page_size - 1) // self.page_size
|
||||
|
||||
|
||||
# Query handler base
|
||||
T = TypeVar('T', bound=Query)
|
||||
R = TypeVar('R')
|
||||
|
||||
class QueryHandler(ABC, Generic[T, R]):
|
||||
@abstractmethod
|
||||
async def handle(self, query: T) -> R:
|
||||
pass
|
||||
|
||||
|
||||
# Query bus
|
||||
class QueryBus:
|
||||
def __init__(self):
|
||||
self._handlers: Dict[Type[Query], QueryHandler] = {}
|
||||
|
||||
def register(self, query_type: Type[Query], handler: QueryHandler):
|
||||
self._handlers[query_type] = handler
|
||||
|
||||
async def dispatch(self, query: Query) -> Any:
|
||||
handler = self._handlers.get(type(query))
|
||||
if not handler:
|
||||
raise ValueError(f"No handler for {type(query).__name__}")
|
||||
return await handler.handle(query)
|
||||
|
||||
|
||||
# Query handler implementation
|
||||
class GetOrderByIdHandler(QueryHandler[GetOrderById, Optional[OrderView]]):
|
||||
def __init__(self, read_db):
|
||||
self.read_db = read_db
|
||||
|
||||
async def handle(self, query: GetOrderById) -> Optional[OrderView]:
|
||||
async with self.read_db.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
SELECT order_id, customer_id, status, total_amount,
|
||||
item_count, created_at, shipped_at
|
||||
FROM order_views
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
query.order_id
|
||||
)
|
||||
if row:
|
||||
return OrderView(**dict(row))
|
||||
return None
|
||||
|
||||
|
||||
class GetCustomerOrdersHandler(QueryHandler[GetCustomerOrders, PaginatedResult[OrderView]]):
|
||||
def __init__(self, read_db):
|
||||
self.read_db = read_db
|
||||
|
||||
async def handle(self, query: GetCustomerOrders) -> PaginatedResult[OrderView]:
|
||||
async with self.read_db.acquire() as conn:
|
||||
# Build query with optional status filter
|
||||
where_clause = "customer_id = $1"
|
||||
params = [query.customer_id]
|
||||
|
||||
if query.status:
|
||||
where_clause += " AND status = $2"
|
||||
params.append(query.status)
|
||||
|
||||
# Get total count
|
||||
total = await conn.fetchval(
|
||||
f"SELECT COUNT(*) FROM order_views WHERE {where_clause}",
|
||||
*params
|
||||
)
|
||||
|
||||
# Get paginated results
|
||||
offset = (query.page - 1) * query.page_size
|
||||
rows = await conn.fetch(
|
||||
f"""
|
||||
SELECT order_id, customer_id, status, total_amount,
|
||||
item_count, created_at, shipped_at
|
||||
FROM order_views
|
||||
WHERE {where_clause}
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}
|
||||
""",
|
||||
*params, query.page_size, offset
|
||||
)
|
||||
|
||||
return PaginatedResult(
|
||||
items=[OrderView(**dict(row)) for row in rows],
|
||||
total=total,
|
||||
page=query.page,
|
||||
page_size=query.page_size
|
||||
)
|
||||
```
|
||||
|
||||
### Template 3: FastAPI CQRS Application
|
||||
|
||||
```python
|
||||
from fastapi import FastAPI, HTTPException, Depends
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Request/Response models
|
||||
class CreateOrderRequest(BaseModel):
|
||||
customer_id: str
|
||||
items: List[dict]
|
||||
shipping_address: dict
|
||||
|
||||
|
||||
class OrderResponse(BaseModel):
|
||||
order_id: str
|
||||
customer_id: str
|
||||
status: str
|
||||
total_amount: float
|
||||
item_count: int
|
||||
created_at: datetime
|
||||
|
||||
|
||||
# Dependency injection
|
||||
def get_command_bus() -> CommandBus:
|
||||
return app.state.command_bus
|
||||
|
||||
|
||||
def get_query_bus() -> QueryBus:
|
||||
return app.state.query_bus
|
||||
|
||||
|
||||
# Command endpoints (POST, PUT, DELETE)
|
||||
@app.post("/orders", response_model=dict)
|
||||
async def create_order(
|
||||
request: CreateOrderRequest,
|
||||
command_bus: CommandBus = Depends(get_command_bus)
|
||||
):
|
||||
command = CreateOrder(
|
||||
customer_id=request.customer_id,
|
||||
items=request.items,
|
||||
shipping_address=request.shipping_address
|
||||
)
|
||||
order_id = await command_bus.dispatch(command)
|
||||
return {"order_id": order_id}
|
||||
|
||||
|
||||
@app.post("/orders/{order_id}/items")
|
||||
async def add_item(
|
||||
order_id: str,
|
||||
product_id: str,
|
||||
quantity: int,
|
||||
price: float,
|
||||
command_bus: CommandBus = Depends(get_command_bus)
|
||||
):
|
||||
command = AddOrderItem(
|
||||
order_id=order_id,
|
||||
product_id=product_id,
|
||||
quantity=quantity,
|
||||
price=price
|
||||
)
|
||||
await command_bus.dispatch(command)
|
||||
return {"status": "item_added"}
|
||||
|
||||
|
||||
@app.delete("/orders/{order_id}")
|
||||
async def cancel_order(
|
||||
order_id: str,
|
||||
reason: str,
|
||||
command_bus: CommandBus = Depends(get_command_bus)
|
||||
):
|
||||
command = CancelOrder(order_id=order_id, reason=reason)
|
||||
await command_bus.dispatch(command)
|
||||
return {"status": "cancelled"}
|
||||
|
||||
|
||||
# Query endpoints (GET)
|
||||
@app.get("/orders/{order_id}", response_model=OrderResponse)
|
||||
async def get_order(
|
||||
order_id: str,
|
||||
query_bus: QueryBus = Depends(get_query_bus)
|
||||
):
|
||||
query = GetOrderById(order_id=order_id)
|
||||
result = await query_bus.dispatch(query)
|
||||
if not result:
|
||||
raise HTTPException(status_code=404, detail="Order not found")
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/customers/{customer_id}/orders")
|
||||
async def get_customer_orders(
|
||||
customer_id: str,
|
||||
status: Optional[str] = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
query_bus: QueryBus = Depends(get_query_bus)
|
||||
):
|
||||
query = GetCustomerOrders(
|
||||
customer_id=customer_id,
|
||||
status=status,
|
||||
page=page,
|
||||
page_size=page_size
|
||||
)
|
||||
return await query_bus.dispatch(query)
|
||||
|
||||
|
||||
@app.get("/orders/search")
|
||||
async def search_orders(
|
||||
q: str,
|
||||
sort_by: str = "created_at",
|
||||
query_bus: QueryBus = Depends(get_query_bus)
|
||||
):
|
||||
query = SearchOrders(query=q, sort_by=sort_by)
|
||||
return await query_bus.dispatch(query)
|
||||
```
|
||||
|
||||
### Template 4: Read Model Synchronization
|
||||
|
||||
```python
|
||||
class ReadModelSynchronizer:
|
||||
"""Keeps read models in sync with events."""
|
||||
|
||||
def __init__(self, event_store, read_db, projections: List[Projection]):
|
||||
self.event_store = event_store
|
||||
self.read_db = read_db
|
||||
self.projections = {p.name: p for p in projections}
|
||||
|
||||
async def run(self):
|
||||
"""Continuously sync read models."""
|
||||
while True:
|
||||
for name, projection in self.projections.items():
|
||||
await self._sync_projection(projection)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
async def _sync_projection(self, projection: Projection):
|
||||
checkpoint = await self._get_checkpoint(projection.name)
|
||||
|
||||
events = await self.event_store.read_all(
|
||||
from_position=checkpoint,
|
||||
limit=100
|
||||
)
|
||||
|
||||
for event in events:
|
||||
if event.event_type in projection.handles():
|
||||
try:
|
||||
await projection.apply(event)
|
||||
except Exception as e:
|
||||
# Log error, possibly retry or skip
|
||||
logger.error(f"Projection error: {e}")
|
||||
continue
|
||||
|
||||
await self._save_checkpoint(projection.name, event.global_position)
|
||||
|
||||
async def rebuild_projection(self, projection_name: str):
|
||||
"""Rebuild a projection from scratch."""
|
||||
projection = self.projections[projection_name]
|
||||
|
||||
# Clear existing data
|
||||
await projection.clear()
|
||||
|
||||
# Reset checkpoint
|
||||
await self._save_checkpoint(projection_name, 0)
|
||||
|
||||
# Rebuild
|
||||
while True:
|
||||
checkpoint = await self._get_checkpoint(projection_name)
|
||||
events = await self.event_store.read_all(checkpoint, 1000)
|
||||
|
||||
if not events:
|
||||
break
|
||||
|
||||
for event in events:
|
||||
if event.event_type in projection.handles():
|
||||
await projection.apply(event)
|
||||
|
||||
await self._save_checkpoint(
|
||||
projection_name,
|
||||
events[-1].global_position
|
||||
)
|
||||
```
|
||||
|
||||
### Template 5: Eventual Consistency Handling
|
||||
|
||||
```python
|
||||
class ConsistentQueryHandler:
|
||||
"""Query handler that can wait for consistency."""
|
||||
|
||||
def __init__(self, read_db, event_store):
|
||||
self.read_db = read_db
|
||||
self.event_store = event_store
|
||||
|
||||
async def query_after_command(
|
||||
self,
|
||||
query: Query,
|
||||
expected_version: int,
|
||||
stream_id: str,
|
||||
timeout: float = 5.0
|
||||
):
|
||||
"""
|
||||
Execute query, ensuring read model is at expected version.
|
||||
Used for read-your-writes consistency.
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
# Check if read model is caught up
|
||||
projection_version = await self._get_projection_version(stream_id)
|
||||
|
||||
if projection_version >= expected_version:
|
||||
return await self.execute_query(query)
|
||||
|
||||
# Wait a bit and retry
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
# Timeout - return stale data with warning
|
||||
return {
|
||||
"data": await self.execute_query(query),
|
||||
"_warning": "Data may be stale"
|
||||
}
|
||||
|
||||
async def _get_projection_version(self, stream_id: str) -> int:
|
||||
"""Get the last processed event version for a stream."""
|
||||
async with self.read_db.acquire() as conn:
|
||||
return await conn.fetchval(
|
||||
"SELECT last_event_version FROM projection_state WHERE stream_id = $1",
|
||||
stream_id
|
||||
) or 0
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Do's
|
||||
- **Separate command and query models** - Different needs
|
||||
- **Use eventual consistency** - Accept propagation delay
|
||||
- **Validate in command handlers** - Before state change
|
||||
- **Denormalize read models** - Optimize for queries
|
||||
- **Version your events** - For schema evolution
|
||||
|
||||
### Don'ts
|
||||
- **Don't query in commands** - Use only for writes
|
||||
- **Don't couple read/write schemas** - Independent evolution
|
||||
- **Don't over-engineer** - Start simple
|
||||
- **Don't ignore consistency SLAs** - Define acceptable lag
|
||||
|
||||
## Resources
|
||||
|
||||
- [CQRS Pattern](https://martinfowler.com/bliki/CQRS.html)
|
||||
- [Microsoft CQRS Guidance](https://docs.microsoft.com/en-us/azure/architecture/patterns/cqrs)
|
||||
435
plugins/backend-development/skills/event-store-design/SKILL.md
Normal file
435
plugins/backend-development/skills/event-store-design/SKILL.md
Normal file
@@ -0,0 +1,435 @@
|
||||
---
|
||||
name: event-store-design
|
||||
description: Design and implement event stores for event-sourced systems. Use when building event sourcing infrastructure, choosing event store technologies, or implementing event persistence patterns.
|
||||
---
|
||||
|
||||
# Event Store Design
|
||||
|
||||
Comprehensive guide to designing event stores for event-sourced applications.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Designing event sourcing infrastructure
|
||||
- Choosing between event store technologies
|
||||
- Implementing custom event stores
|
||||
- Optimizing event storage and retrieval
|
||||
- Setting up event store schemas
|
||||
- Planning for event store scaling
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### 1. Event Store Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────┐
|
||||
│ Event Store │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ Stream 1 │ │ Stream 2 │ │ Stream 3 │ │
|
||||
│ │ (Aggregate) │ │ (Aggregate) │ │ (Aggregate) │ │
|
||||
│ ├─────────────┤ ├─────────────┤ ├─────────────┤ │
|
||||
│ │ Event 1 │ │ Event 1 │ │ Event 1 │ │
|
||||
│ │ Event 2 │ │ Event 2 │ │ Event 2 │ │
|
||||
│ │ Event 3 │ │ ... │ │ Event 3 │ │
|
||||
│ │ ... │ │ │ │ Event 4 │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────┘ │
|
||||
├─────────────────────────────────────────────────────┤
|
||||
│ Global Position: 1 → 2 → 3 → 4 → 5 → 6 → ... │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2. Event Store Requirements
|
||||
|
||||
| Requirement | Description |
|
||||
|-------------|-------------|
|
||||
| **Append-only** | Events are immutable, only appends |
|
||||
| **Ordered** | Per-stream and global ordering |
|
||||
| **Versioned** | Optimistic concurrency control |
|
||||
| **Subscriptions** | Real-time event notifications |
|
||||
| **Idempotent** | Handle duplicate writes safely |
|
||||
|
||||
## Technology Comparison
|
||||
|
||||
| Technology | Best For | Limitations |
|
||||
|------------|----------|-------------|
|
||||
| **EventStoreDB** | Pure event sourcing | Single-purpose |
|
||||
| **PostgreSQL** | Existing Postgres stack | Manual implementation |
|
||||
| **Kafka** | High-throughput streaming | Not ideal for per-stream queries |
|
||||
| **DynamoDB** | Serverless, AWS-native | Query limitations |
|
||||
| **Marten** | .NET ecosystems | .NET specific |
|
||||
|
||||
## Templates
|
||||
|
||||
### Template 1: PostgreSQL Event Store Schema
|
||||
|
||||
```sql
|
||||
-- Events table
|
||||
CREATE TABLE events (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
stream_id VARCHAR(255) NOT NULL,
|
||||
stream_type VARCHAR(255) NOT NULL,
|
||||
event_type VARCHAR(255) NOT NULL,
|
||||
event_data JSONB NOT NULL,
|
||||
metadata JSONB DEFAULT '{}',
|
||||
version BIGINT NOT NULL,
|
||||
global_position BIGSERIAL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
|
||||
CONSTRAINT unique_stream_version UNIQUE (stream_id, version)
|
||||
);
|
||||
|
||||
-- Index for stream queries
|
||||
CREATE INDEX idx_events_stream_id ON events(stream_id, version);
|
||||
|
||||
-- Index for global subscription
|
||||
CREATE INDEX idx_events_global_position ON events(global_position);
|
||||
|
||||
-- Index for event type queries
|
||||
CREATE INDEX idx_events_event_type ON events(event_type);
|
||||
|
||||
-- Index for time-based queries
|
||||
CREATE INDEX idx_events_created_at ON events(created_at);
|
||||
|
||||
-- Snapshots table
|
||||
CREATE TABLE snapshots (
|
||||
stream_id VARCHAR(255) PRIMARY KEY,
|
||||
stream_type VARCHAR(255) NOT NULL,
|
||||
snapshot_data JSONB NOT NULL,
|
||||
version BIGINT NOT NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Subscriptions checkpoint table
|
||||
CREATE TABLE subscription_checkpoints (
|
||||
subscription_id VARCHAR(255) PRIMARY KEY,
|
||||
last_position BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
```
|
||||
|
||||
### Template 2: Python Event Store Implementation
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, List
|
||||
from uuid import UUID, uuid4
|
||||
import json
|
||||
import asyncpg
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
stream_id: str
|
||||
event_type: str
|
||||
data: dict
|
||||
metadata: dict = field(default_factory=dict)
|
||||
event_id: UUID = field(default_factory=uuid4)
|
||||
version: Optional[int] = None
|
||||
global_position: Optional[int] = None
|
||||
created_at: datetime = field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class EventStore:
|
||||
def __init__(self, pool: asyncpg.Pool):
|
||||
self.pool = pool
|
||||
|
||||
async def append_events(
|
||||
self,
|
||||
stream_id: str,
|
||||
stream_type: str,
|
||||
events: List[Event],
|
||||
expected_version: Optional[int] = None
|
||||
) -> List[Event]:
|
||||
"""Append events to a stream with optimistic concurrency."""
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
# Check expected version
|
||||
if expected_version is not None:
|
||||
current = await conn.fetchval(
|
||||
"SELECT MAX(version) FROM events WHERE stream_id = $1",
|
||||
stream_id
|
||||
)
|
||||
current = current or 0
|
||||
if current != expected_version:
|
||||
raise ConcurrencyError(
|
||||
f"Expected version {expected_version}, got {current}"
|
||||
)
|
||||
|
||||
# Get starting version
|
||||
start_version = await conn.fetchval(
|
||||
"SELECT COALESCE(MAX(version), 0) + 1 FROM events WHERE stream_id = $1",
|
||||
stream_id
|
||||
)
|
||||
|
||||
# Insert events
|
||||
saved_events = []
|
||||
for i, event in enumerate(events):
|
||||
event.version = start_version + i
|
||||
row = await conn.fetchrow(
|
||||
"""
|
||||
INSERT INTO events (id, stream_id, stream_type, event_type,
|
||||
event_data, metadata, version, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING global_position
|
||||
""",
|
||||
event.event_id,
|
||||
stream_id,
|
||||
stream_type,
|
||||
event.event_type,
|
||||
json.dumps(event.data),
|
||||
json.dumps(event.metadata),
|
||||
event.version,
|
||||
event.created_at
|
||||
)
|
||||
event.global_position = row['global_position']
|
||||
saved_events.append(event)
|
||||
|
||||
return saved_events
|
||||
|
||||
async def read_stream(
|
||||
self,
|
||||
stream_id: str,
|
||||
from_version: int = 0,
|
||||
limit: int = 1000
|
||||
) -> List[Event]:
|
||||
"""Read events from a stream."""
|
||||
async with self.pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, stream_id, event_type, event_data, metadata,
|
||||
version, global_position, created_at
|
||||
FROM events
|
||||
WHERE stream_id = $1 AND version >= $2
|
||||
ORDER BY version
|
||||
LIMIT $3
|
||||
""",
|
||||
stream_id, from_version, limit
|
||||
)
|
||||
return [self._row_to_event(row) for row in rows]
|
||||
|
||||
async def read_all(
|
||||
self,
|
||||
from_position: int = 0,
|
||||
limit: int = 1000
|
||||
) -> List[Event]:
|
||||
"""Read all events globally."""
|
||||
async with self.pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT id, stream_id, event_type, event_data, metadata,
|
||||
version, global_position, created_at
|
||||
FROM events
|
||||
WHERE global_position > $1
|
||||
ORDER BY global_position
|
||||
LIMIT $2
|
||||
""",
|
||||
from_position, limit
|
||||
)
|
||||
return [self._row_to_event(row) for row in rows]
|
||||
|
||||
async def subscribe(
|
||||
self,
|
||||
subscription_id: str,
|
||||
handler,
|
||||
from_position: int = 0,
|
||||
batch_size: int = 100
|
||||
):
|
||||
"""Subscribe to all events from a position."""
|
||||
# Get checkpoint
|
||||
async with self.pool.acquire() as conn:
|
||||
checkpoint = await conn.fetchval(
|
||||
"""
|
||||
SELECT last_position FROM subscription_checkpoints
|
||||
WHERE subscription_id = $1
|
||||
""",
|
||||
subscription_id
|
||||
)
|
||||
position = checkpoint or from_position
|
||||
|
||||
while True:
|
||||
events = await self.read_all(position, batch_size)
|
||||
if not events:
|
||||
await asyncio.sleep(1) # Poll interval
|
||||
continue
|
||||
|
||||
for event in events:
|
||||
await handler(event)
|
||||
position = event.global_position
|
||||
|
||||
# Save checkpoint
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO subscription_checkpoints (subscription_id, last_position)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (subscription_id)
|
||||
DO UPDATE SET last_position = $2, updated_at = NOW()
|
||||
""",
|
||||
subscription_id, position
|
||||
)
|
||||
|
||||
def _row_to_event(self, row) -> Event:
|
||||
return Event(
|
||||
event_id=row['id'],
|
||||
stream_id=row['stream_id'],
|
||||
event_type=row['event_type'],
|
||||
data=json.loads(row['event_data']),
|
||||
metadata=json.loads(row['metadata']),
|
||||
version=row['version'],
|
||||
global_position=row['global_position'],
|
||||
created_at=row['created_at']
|
||||
)
|
||||
|
||||
|
||||
class ConcurrencyError(Exception):
|
||||
"""Raised when optimistic concurrency check fails."""
|
||||
pass
|
||||
```
|
||||
|
||||
### Template 3: EventStoreDB Usage
|
||||
|
||||
```python
|
||||
from esdbclient import EventStoreDBClient, NewEvent, StreamState
|
||||
import json
|
||||
|
||||
# Connect
|
||||
client = EventStoreDBClient(uri="esdb://localhost:2113?tls=false")
|
||||
|
||||
# Append events
|
||||
def append_events(stream_name: str, events: list, expected_revision=None):
|
||||
new_events = [
|
||||
NewEvent(
|
||||
type=event['type'],
|
||||
data=json.dumps(event['data']).encode(),
|
||||
metadata=json.dumps(event.get('metadata', {})).encode()
|
||||
)
|
||||
for event in events
|
||||
]
|
||||
|
||||
if expected_revision is None:
|
||||
state = StreamState.ANY
|
||||
elif expected_revision == -1:
|
||||
state = StreamState.NO_STREAM
|
||||
else:
|
||||
state = expected_revision
|
||||
|
||||
return client.append_to_stream(
|
||||
stream_name=stream_name,
|
||||
events=new_events,
|
||||
current_version=state
|
||||
)
|
||||
|
||||
# Read stream
|
||||
def read_stream(stream_name: str, from_revision: int = 0):
|
||||
events = client.get_stream(
|
||||
stream_name=stream_name,
|
||||
stream_position=from_revision
|
||||
)
|
||||
return [
|
||||
{
|
||||
'type': event.type,
|
||||
'data': json.loads(event.data),
|
||||
'metadata': json.loads(event.metadata) if event.metadata else {},
|
||||
'stream_position': event.stream_position,
|
||||
'commit_position': event.commit_position
|
||||
}
|
||||
for event in events
|
||||
]
|
||||
|
||||
# Subscribe to all
|
||||
async def subscribe_to_all(handler, from_position: int = 0):
|
||||
subscription = client.subscribe_to_all(commit_position=from_position)
|
||||
async for event in subscription:
|
||||
await handler({
|
||||
'type': event.type,
|
||||
'data': json.loads(event.data),
|
||||
'stream_id': event.stream_name,
|
||||
'position': event.commit_position
|
||||
})
|
||||
|
||||
# Category projection ($ce-Category)
|
||||
def read_category(category: str):
|
||||
"""Read all events for a category using system projection."""
|
||||
return read_stream(f"$ce-{category}")
|
||||
```
|
||||
|
||||
### Template 4: DynamoDB Event Store
|
||||
|
||||
```python
|
||||
import boto3
|
||||
from boto3.dynamodb.conditions import Key
|
||||
from datetime import datetime
|
||||
import json
|
||||
import uuid
|
||||
|
||||
class DynamoEventStore:
|
||||
def __init__(self, table_name: str):
|
||||
self.dynamodb = boto3.resource('dynamodb')
|
||||
self.table = self.dynamodb.Table(table_name)
|
||||
|
||||
def append_events(self, stream_id: str, events: list, expected_version: int = None):
|
||||
"""Append events with conditional write for concurrency."""
|
||||
with self.table.batch_writer() as batch:
|
||||
for i, event in enumerate(events):
|
||||
version = (expected_version or 0) + i + 1
|
||||
item = {
|
||||
'PK': f"STREAM#{stream_id}",
|
||||
'SK': f"VERSION#{version:020d}",
|
||||
'GSI1PK': 'EVENTS',
|
||||
'GSI1SK': datetime.utcnow().isoformat(),
|
||||
'event_id': str(uuid.uuid4()),
|
||||
'stream_id': stream_id,
|
||||
'event_type': event['type'],
|
||||
'event_data': json.dumps(event['data']),
|
||||
'version': version,
|
||||
'created_at': datetime.utcnow().isoformat()
|
||||
}
|
||||
batch.put_item(Item=item)
|
||||
return events
|
||||
|
||||
def read_stream(self, stream_id: str, from_version: int = 0):
|
||||
"""Read events from a stream."""
|
||||
response = self.table.query(
|
||||
KeyConditionExpression=Key('PK').eq(f"STREAM#{stream_id}") &
|
||||
Key('SK').gte(f"VERSION#{from_version:020d}")
|
||||
)
|
||||
return [
|
||||
{
|
||||
'event_type': item['event_type'],
|
||||
'data': json.loads(item['event_data']),
|
||||
'version': item['version']
|
||||
}
|
||||
for item in response['Items']
|
||||
]
|
||||
|
||||
# Table definition (CloudFormation/Terraform)
|
||||
"""
|
||||
DynamoDB Table:
|
||||
- PK (Partition Key): String
|
||||
- SK (Sort Key): String
|
||||
- GSI1PK, GSI1SK for global ordering
|
||||
|
||||
Capacity: On-demand or provisioned based on throughput needs
|
||||
"""
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Do's
|
||||
- **Use stream IDs that include aggregate type** - `Order-{uuid}`
|
||||
- **Include correlation/causation IDs** - For tracing
|
||||
- **Version events from day one** - Plan for schema evolution
|
||||
- **Implement idempotency** - Use event IDs for deduplication
|
||||
- **Index appropriately** - For your query patterns
|
||||
|
||||
### Don'ts
|
||||
- **Don't update or delete events** - They're immutable facts
|
||||
- **Don't store large payloads** - Keep events small
|
||||
- **Don't skip optimistic concurrency** - Prevents data corruption
|
||||
- **Don't ignore backpressure** - Handle slow consumers
|
||||
|
||||
## Resources
|
||||
|
||||
- [EventStoreDB](https://www.eventstore.com/)
|
||||
- [Marten Events](https://martendb.io/events/)
|
||||
- [Event Sourcing Pattern](https://docs.microsoft.com/en-us/azure/architecture/patterns/event-sourcing)
|
||||
488
plugins/backend-development/skills/projection-patterns/SKILL.md
Normal file
488
plugins/backend-development/skills/projection-patterns/SKILL.md
Normal file
@@ -0,0 +1,488 @@
|
||||
---
|
||||
name: projection-patterns
|
||||
description: Build read models and projections from event streams. Use when implementing CQRS read sides, building materialized views, or optimizing query performance in event-sourced systems.
|
||||
---
|
||||
|
||||
# Projection Patterns
|
||||
|
||||
Comprehensive guide to building projections and read models for event-sourced systems.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Building CQRS read models
|
||||
- Creating materialized views from events
|
||||
- Optimizing query performance
|
||||
- Implementing real-time dashboards
|
||||
- Building search indexes from events
|
||||
- Aggregating data across streams
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### 1. Projection Architecture
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ Event Store │────►│ Projector │────►│ Read Model │
|
||||
│ │ │ │ │ (Database) │
|
||||
│ ┌─────────┐ │ │ ┌─────────┐ │ │ ┌─────────┐ │
|
||||
│ │ Events │ │ │ │ Handler │ │ │ │ Tables │ │
|
||||
│ └─────────┘ │ │ │ Logic │ │ │ │ Views │ │
|
||||
│ │ │ └─────────┘ │ │ │ Cache │ │
|
||||
└─────────────┘ └─────────────┘ └─────────────┘
|
||||
```
|
||||
|
||||
### 2. Projection Types
|
||||
|
||||
| Type | Description | Use Case |
|
||||
|------|-------------|----------|
|
||||
| **Live** | Real-time from subscription | Current state queries |
|
||||
| **Catchup** | Process historical events | Rebuilding read models |
|
||||
| **Persistent** | Stores checkpoint | Resume after restart |
|
||||
| **Inline** | Same transaction as write | Strong consistency |
|
||||
|
||||
## Templates
|
||||
|
||||
### Template 1: Basic Projector
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Any, Callable, List
|
||||
import asyncpg
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
stream_id: str
|
||||
event_type: str
|
||||
data: dict
|
||||
version: int
|
||||
global_position: int
|
||||
|
||||
|
||||
class Projection(ABC):
|
||||
"""Base class for projections."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Unique projection name for checkpointing."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def handles(self) -> List[str]:
|
||||
"""List of event types this projection handles."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
async def apply(self, event: Event) -> None:
|
||||
"""Apply event to the read model."""
|
||||
pass
|
||||
|
||||
|
||||
class Projector:
|
||||
"""Runs projections from event store."""
|
||||
|
||||
def __init__(self, event_store, checkpoint_store):
|
||||
self.event_store = event_store
|
||||
self.checkpoint_store = checkpoint_store
|
||||
self.projections: List[Projection] = []
|
||||
|
||||
def register(self, projection: Projection):
|
||||
self.projections.append(projection)
|
||||
|
||||
async def run(self, batch_size: int = 100):
|
||||
"""Run all projections continuously."""
|
||||
while True:
|
||||
for projection in self.projections:
|
||||
await self._run_projection(projection, batch_size)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
async def _run_projection(self, projection: Projection, batch_size: int):
|
||||
checkpoint = await self.checkpoint_store.get(projection.name)
|
||||
position = checkpoint or 0
|
||||
|
||||
events = await self.event_store.read_all(position, batch_size)
|
||||
|
||||
for event in events:
|
||||
if event.event_type in projection.handles():
|
||||
await projection.apply(event)
|
||||
|
||||
await self.checkpoint_store.save(
|
||||
projection.name,
|
||||
event.global_position
|
||||
)
|
||||
|
||||
async def rebuild(self, projection: Projection):
|
||||
"""Rebuild a projection from scratch."""
|
||||
await self.checkpoint_store.delete(projection.name)
|
||||
# Optionally clear read model tables
|
||||
await self._run_projection(projection, batch_size=1000)
|
||||
```
|
||||
|
||||
### Template 2: Order Summary Projection
|
||||
|
||||
```python
|
||||
class OrderSummaryProjection(Projection):
|
||||
"""Projects order events to a summary read model."""
|
||||
|
||||
def __init__(self, db_pool: asyncpg.Pool):
|
||||
self.pool = db_pool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "order_summary"
|
||||
|
||||
def handles(self) -> List[str]:
|
||||
return [
|
||||
"OrderCreated",
|
||||
"OrderItemAdded",
|
||||
"OrderItemRemoved",
|
||||
"OrderShipped",
|
||||
"OrderCompleted",
|
||||
"OrderCancelled"
|
||||
]
|
||||
|
||||
async def apply(self, event: Event) -> None:
|
||||
handlers = {
|
||||
"OrderCreated": self._handle_created,
|
||||
"OrderItemAdded": self._handle_item_added,
|
||||
"OrderItemRemoved": self._handle_item_removed,
|
||||
"OrderShipped": self._handle_shipped,
|
||||
"OrderCompleted": self._handle_completed,
|
||||
"OrderCancelled": self._handle_cancelled,
|
||||
}
|
||||
|
||||
handler = handlers.get(event.event_type)
|
||||
if handler:
|
||||
await handler(event)
|
||||
|
||||
async def _handle_created(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO order_summaries
|
||||
(order_id, customer_id, status, total_amount, item_count, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['customer_id'],
|
||||
'pending',
|
||||
0,
|
||||
0,
|
||||
event.data['created_at']
|
||||
)
|
||||
|
||||
async def _handle_item_added(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE order_summaries
|
||||
SET total_amount = total_amount + $2,
|
||||
item_count = item_count + 1,
|
||||
updated_at = NOW()
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['price'] * event.data['quantity']
|
||||
)
|
||||
|
||||
async def _handle_item_removed(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE order_summaries
|
||||
SET total_amount = total_amount - $2,
|
||||
item_count = item_count - 1,
|
||||
updated_at = NOW()
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['price'] * event.data['quantity']
|
||||
)
|
||||
|
||||
async def _handle_shipped(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE order_summaries
|
||||
SET status = 'shipped',
|
||||
shipped_at = $2,
|
||||
updated_at = NOW()
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['shipped_at']
|
||||
)
|
||||
|
||||
async def _handle_completed(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE order_summaries
|
||||
SET status = 'completed',
|
||||
completed_at = $2,
|
||||
updated_at = NOW()
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['completed_at']
|
||||
)
|
||||
|
||||
async def _handle_cancelled(self, event: Event):
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE order_summaries
|
||||
SET status = 'cancelled',
|
||||
cancelled_at = $2,
|
||||
cancellation_reason = $3,
|
||||
updated_at = NOW()
|
||||
WHERE order_id = $1
|
||||
""",
|
||||
event.data['order_id'],
|
||||
event.data['cancelled_at'],
|
||||
event.data.get('reason')
|
||||
)
|
||||
```
|
||||
|
||||
### Template 3: Elasticsearch Search Projection
|
||||
|
||||
```python
|
||||
from elasticsearch import AsyncElasticsearch
|
||||
|
||||
class ProductSearchProjection(Projection):
|
||||
"""Projects product events to Elasticsearch for full-text search."""
|
||||
|
||||
def __init__(self, es_client: AsyncElasticsearch):
|
||||
self.es = es_client
|
||||
self.index = "products"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "product_search"
|
||||
|
||||
def handles(self) -> List[str]:
|
||||
return [
|
||||
"ProductCreated",
|
||||
"ProductUpdated",
|
||||
"ProductPriceChanged",
|
||||
"ProductDeleted"
|
||||
]
|
||||
|
||||
async def apply(self, event: Event) -> None:
|
||||
if event.event_type == "ProductCreated":
|
||||
await self.es.index(
|
||||
index=self.index,
|
||||
id=event.data['product_id'],
|
||||
document={
|
||||
'name': event.data['name'],
|
||||
'description': event.data['description'],
|
||||
'category': event.data['category'],
|
||||
'price': event.data['price'],
|
||||
'tags': event.data.get('tags', []),
|
||||
'created_at': event.data['created_at']
|
||||
}
|
||||
)
|
||||
|
||||
elif event.event_type == "ProductUpdated":
|
||||
await self.es.update(
|
||||
index=self.index,
|
||||
id=event.data['product_id'],
|
||||
doc={
|
||||
'name': event.data['name'],
|
||||
'description': event.data['description'],
|
||||
'category': event.data['category'],
|
||||
'tags': event.data.get('tags', []),
|
||||
'updated_at': event.data['updated_at']
|
||||
}
|
||||
)
|
||||
|
||||
elif event.event_type == "ProductPriceChanged":
|
||||
await self.es.update(
|
||||
index=self.index,
|
||||
id=event.data['product_id'],
|
||||
doc={
|
||||
'price': event.data['new_price'],
|
||||
'price_updated_at': event.data['changed_at']
|
||||
}
|
||||
)
|
||||
|
||||
elif event.event_type == "ProductDeleted":
|
||||
await self.es.delete(
|
||||
index=self.index,
|
||||
id=event.data['product_id']
|
||||
)
|
||||
```
|
||||
|
||||
### Template 4: Aggregating Projection
|
||||
|
||||
```python
|
||||
class DailySalesProjection(Projection):
|
||||
"""Aggregates sales data by day for reporting."""
|
||||
|
||||
def __init__(self, db_pool: asyncpg.Pool):
|
||||
self.pool = db_pool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "daily_sales"
|
||||
|
||||
def handles(self) -> List[str]:
|
||||
return ["OrderCompleted", "OrderRefunded"]
|
||||
|
||||
async def apply(self, event: Event) -> None:
|
||||
if event.event_type == "OrderCompleted":
|
||||
await self._increment_sales(event)
|
||||
elif event.event_type == "OrderRefunded":
|
||||
await self._decrement_sales(event)
|
||||
|
||||
async def _increment_sales(self, event: Event):
|
||||
date = event.data['completed_at'][:10] # YYYY-MM-DD
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO daily_sales (date, total_orders, total_revenue, total_items)
|
||||
VALUES ($1, 1, $2, $3)
|
||||
ON CONFLICT (date) DO UPDATE SET
|
||||
total_orders = daily_sales.total_orders + 1,
|
||||
total_revenue = daily_sales.total_revenue + $2,
|
||||
total_items = daily_sales.total_items + $3,
|
||||
updated_at = NOW()
|
||||
""",
|
||||
date,
|
||||
event.data['total_amount'],
|
||||
event.data['item_count']
|
||||
)
|
||||
|
||||
async def _decrement_sales(self, event: Event):
|
||||
date = event.data['original_completed_at'][:10]
|
||||
async with self.pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE daily_sales SET
|
||||
total_orders = total_orders - 1,
|
||||
total_revenue = total_revenue - $2,
|
||||
total_refunds = total_refunds + $2,
|
||||
updated_at = NOW()
|
||||
WHERE date = $1
|
||||
""",
|
||||
date,
|
||||
event.data['refund_amount']
|
||||
)
|
||||
```
|
||||
|
||||
### Template 5: Multi-Table Projection
|
||||
|
||||
```python
|
||||
class CustomerActivityProjection(Projection):
|
||||
"""Projects customer activity across multiple tables."""
|
||||
|
||||
def __init__(self, db_pool: asyncpg.Pool):
|
||||
self.pool = db_pool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "customer_activity"
|
||||
|
||||
def handles(self) -> List[str]:
|
||||
return [
|
||||
"CustomerCreated",
|
||||
"OrderCompleted",
|
||||
"ReviewSubmitted",
|
||||
"CustomerTierChanged"
|
||||
]
|
||||
|
||||
async def apply(self, event: Event) -> None:
|
||||
async with self.pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
if event.event_type == "CustomerCreated":
|
||||
# Insert into customers table
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO customers (customer_id, email, name, tier, created_at)
|
||||
VALUES ($1, $2, $3, 'bronze', $4)
|
||||
""",
|
||||
event.data['customer_id'],
|
||||
event.data['email'],
|
||||
event.data['name'],
|
||||
event.data['created_at']
|
||||
)
|
||||
# Initialize activity summary
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO customer_activity_summary
|
||||
(customer_id, total_orders, total_spent, total_reviews)
|
||||
VALUES ($1, 0, 0, 0)
|
||||
""",
|
||||
event.data['customer_id']
|
||||
)
|
||||
|
||||
elif event.event_type == "OrderCompleted":
|
||||
# Update activity summary
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE customer_activity_summary SET
|
||||
total_orders = total_orders + 1,
|
||||
total_spent = total_spent + $2,
|
||||
last_order_at = $3
|
||||
WHERE customer_id = $1
|
||||
""",
|
||||
event.data['customer_id'],
|
||||
event.data['total_amount'],
|
||||
event.data['completed_at']
|
||||
)
|
||||
# Insert into order history
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO customer_order_history
|
||||
(customer_id, order_id, amount, completed_at)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
""",
|
||||
event.data['customer_id'],
|
||||
event.data['order_id'],
|
||||
event.data['total_amount'],
|
||||
event.data['completed_at']
|
||||
)
|
||||
|
||||
elif event.event_type == "ReviewSubmitted":
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE customer_activity_summary SET
|
||||
total_reviews = total_reviews + 1,
|
||||
last_review_at = $2
|
||||
WHERE customer_id = $1
|
||||
""",
|
||||
event.data['customer_id'],
|
||||
event.data['submitted_at']
|
||||
)
|
||||
|
||||
elif event.event_type == "CustomerTierChanged":
|
||||
await conn.execute(
|
||||
"""
|
||||
UPDATE customers SET tier = $2, updated_at = NOW()
|
||||
WHERE customer_id = $1
|
||||
""",
|
||||
event.data['customer_id'],
|
||||
event.data['new_tier']
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Do's
|
||||
- **Make projections idempotent** - Safe to replay
|
||||
- **Use transactions** - For multi-table updates
|
||||
- **Store checkpoints** - Resume after failures
|
||||
- **Monitor lag** - Alert on projection delays
|
||||
- **Plan for rebuilds** - Design for reconstruction
|
||||
|
||||
### Don'ts
|
||||
- **Don't couple projections** - Each is independent
|
||||
- **Don't skip error handling** - Log and alert on failures
|
||||
- **Don't ignore ordering** - Events must be processed in order
|
||||
- **Don't over-normalize** - Denormalize for query patterns
|
||||
|
||||
## Resources
|
||||
|
||||
- [CQRS Pattern](https://docs.microsoft.com/en-us/azure/architecture/patterns/cqrs)
|
||||
- [Projection Building Blocks](https://zimarev.com/blog/event-sourcing/projections/)
|
||||
482
plugins/backend-development/skills/saga-orchestration/SKILL.md
Normal file
482
plugins/backend-development/skills/saga-orchestration/SKILL.md
Normal file
@@ -0,0 +1,482 @@
|
||||
---
|
||||
name: saga-orchestration
|
||||
description: Implement saga patterns for distributed transactions and cross-aggregate workflows. Use when coordinating multi-step business processes, handling compensating transactions, or managing long-running workflows.
|
||||
---
|
||||
|
||||
# Saga Orchestration
|
||||
|
||||
Patterns for managing distributed transactions and long-running business processes.
|
||||
|
||||
## When to Use This Skill
|
||||
|
||||
- Coordinating multi-service transactions
|
||||
- Implementing compensating transactions
|
||||
- Managing long-running business workflows
|
||||
- Handling failures in distributed systems
|
||||
- Building order fulfillment processes
|
||||
- Implementing approval workflows
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### 1. Saga Types
|
||||
|
||||
```
|
||||
Choreography Orchestration
|
||||
┌─────┐ ┌─────┐ ┌─────┐ ┌─────────────┐
|
||||
│Svc A│─►│Svc B│─►│Svc C│ │ Orchestrator│
|
||||
└─────┘ └─────┘ └─────┘ └──────┬──────┘
|
||||
│ │ │ │
|
||||
▼ ▼ ▼ ┌─────┼─────┐
|
||||
Event Event Event ▼ ▼ ▼
|
||||
┌────┐┌────┐┌────┐
|
||||
│Svc1││Svc2││Svc3│
|
||||
└────┘└────┘└────┘
|
||||
```
|
||||
|
||||
### 2. Saga Execution States
|
||||
|
||||
| State | Description |
|
||||
|-------|-------------|
|
||||
| **Started** | Saga initiated |
|
||||
| **Pending** | Waiting for step completion |
|
||||
| **Compensating** | Rolling back due to failure |
|
||||
| **Completed** | All steps succeeded |
|
||||
| **Failed** | Saga failed after compensation |
|
||||
|
||||
## Templates
|
||||
|
||||
### Template 1: Saga Orchestrator Base
|
||||
|
||||
```python
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
class SagaState(Enum):
|
||||
STARTED = "started"
|
||||
PENDING = "pending"
|
||||
COMPENSATING = "compensating"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SagaStep:
|
||||
name: str
|
||||
action: str
|
||||
compensation: str
|
||||
status: str = "pending"
|
||||
result: Optional[Dict] = None
|
||||
error: Optional[str] = None
|
||||
executed_at: Optional[datetime] = None
|
||||
compensated_at: Optional[datetime] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Saga:
|
||||
saga_id: str
|
||||
saga_type: str
|
||||
state: SagaState
|
||||
data: Dict[str, Any]
|
||||
steps: List[SagaStep]
|
||||
current_step: int = 0
|
||||
created_at: datetime = field(default_factory=datetime.utcnow)
|
||||
updated_at: datetime = field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class SagaOrchestrator(ABC):
|
||||
"""Base class for saga orchestrators."""
|
||||
|
||||
def __init__(self, saga_store, event_publisher):
|
||||
self.saga_store = saga_store
|
||||
self.event_publisher = event_publisher
|
||||
|
||||
@abstractmethod
|
||||
def define_steps(self, data: Dict) -> List[SagaStep]:
|
||||
"""Define the saga steps."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def saga_type(self) -> str:
|
||||
"""Unique saga type identifier."""
|
||||
pass
|
||||
|
||||
async def start(self, data: Dict) -> Saga:
|
||||
"""Start a new saga."""
|
||||
saga = Saga(
|
||||
saga_id=str(uuid.uuid4()),
|
||||
saga_type=self.saga_type,
|
||||
state=SagaState.STARTED,
|
||||
data=data,
|
||||
steps=self.define_steps(data)
|
||||
)
|
||||
await self.saga_store.save(saga)
|
||||
await self._execute_next_step(saga)
|
||||
return saga
|
||||
|
||||
async def handle_step_completed(self, saga_id: str, step_name: str, result: Dict):
|
||||
"""Handle successful step completion."""
|
||||
saga = await self.saga_store.get(saga_id)
|
||||
|
||||
# Update step
|
||||
for step in saga.steps:
|
||||
if step.name == step_name:
|
||||
step.status = "completed"
|
||||
step.result = result
|
||||
step.executed_at = datetime.utcnow()
|
||||
break
|
||||
|
||||
saga.current_step += 1
|
||||
saga.updated_at = datetime.utcnow()
|
||||
|
||||
# Check if saga is complete
|
||||
if saga.current_step >= len(saga.steps):
|
||||
saga.state = SagaState.COMPLETED
|
||||
await self.saga_store.save(saga)
|
||||
await self._on_saga_completed(saga)
|
||||
else:
|
||||
saga.state = SagaState.PENDING
|
||||
await self.saga_store.save(saga)
|
||||
await self._execute_next_step(saga)
|
||||
|
||||
async def handle_step_failed(self, saga_id: str, step_name: str, error: str):
|
||||
"""Handle step failure - start compensation."""
|
||||
saga = await self.saga_store.get(saga_id)
|
||||
|
||||
# Mark step as failed
|
||||
for step in saga.steps:
|
||||
if step.name == step_name:
|
||||
step.status = "failed"
|
||||
step.error = error
|
||||
break
|
||||
|
||||
saga.state = SagaState.COMPENSATING
|
||||
saga.updated_at = datetime.utcnow()
|
||||
await self.saga_store.save(saga)
|
||||
|
||||
# Start compensation from current step backwards
|
||||
await self._compensate(saga)
|
||||
|
||||
async def _execute_next_step(self, saga: Saga):
|
||||
"""Execute the next step in the saga."""
|
||||
if saga.current_step >= len(saga.steps):
|
||||
return
|
||||
|
||||
step = saga.steps[saga.current_step]
|
||||
step.status = "executing"
|
||||
await self.saga_store.save(saga)
|
||||
|
||||
# Publish command to execute step
|
||||
await self.event_publisher.publish(
|
||||
step.action,
|
||||
{
|
||||
"saga_id": saga.saga_id,
|
||||
"step_name": step.name,
|
||||
**saga.data
|
||||
}
|
||||
)
|
||||
|
||||
async def _compensate(self, saga: Saga):
|
||||
"""Execute compensation for completed steps."""
|
||||
# Compensate in reverse order
|
||||
for i in range(saga.current_step - 1, -1, -1):
|
||||
step = saga.steps[i]
|
||||
if step.status == "completed":
|
||||
step.status = "compensating"
|
||||
await self.saga_store.save(saga)
|
||||
|
||||
await self.event_publisher.publish(
|
||||
step.compensation,
|
||||
{
|
||||
"saga_id": saga.saga_id,
|
||||
"step_name": step.name,
|
||||
"original_result": step.result,
|
||||
**saga.data
|
||||
}
|
||||
)
|
||||
|
||||
async def handle_compensation_completed(self, saga_id: str, step_name: str):
|
||||
"""Handle compensation completion."""
|
||||
saga = await self.saga_store.get(saga_id)
|
||||
|
||||
for step in saga.steps:
|
||||
if step.name == step_name:
|
||||
step.status = "compensated"
|
||||
step.compensated_at = datetime.utcnow()
|
||||
break
|
||||
|
||||
# Check if all compensations complete
|
||||
all_compensated = all(
|
||||
s.status in ("compensated", "pending", "failed")
|
||||
for s in saga.steps
|
||||
)
|
||||
|
||||
if all_compensated:
|
||||
saga.state = SagaState.FAILED
|
||||
await self._on_saga_failed(saga)
|
||||
|
||||
await self.saga_store.save(saga)
|
||||
|
||||
async def _on_saga_completed(self, saga: Saga):
|
||||
"""Called when saga completes successfully."""
|
||||
await self.event_publisher.publish(
|
||||
f"{self.saga_type}Completed",
|
||||
{"saga_id": saga.saga_id, **saga.data}
|
||||
)
|
||||
|
||||
async def _on_saga_failed(self, saga: Saga):
|
||||
"""Called when saga fails after compensation."""
|
||||
await self.event_publisher.publish(
|
||||
f"{self.saga_type}Failed",
|
||||
{"saga_id": saga.saga_id, "error": "Saga failed", **saga.data}
|
||||
)
|
||||
```
|
||||
|
||||
### Template 2: Order Fulfillment Saga
|
||||
|
||||
```python
|
||||
class OrderFulfillmentSaga(SagaOrchestrator):
|
||||
"""Orchestrates order fulfillment across services."""
|
||||
|
||||
@property
|
||||
def saga_type(self) -> str:
|
||||
return "OrderFulfillment"
|
||||
|
||||
def define_steps(self, data: Dict) -> List[SagaStep]:
|
||||
return [
|
||||
SagaStep(
|
||||
name="reserve_inventory",
|
||||
action="InventoryService.ReserveItems",
|
||||
compensation="InventoryService.ReleaseReservation"
|
||||
),
|
||||
SagaStep(
|
||||
name="process_payment",
|
||||
action="PaymentService.ProcessPayment",
|
||||
compensation="PaymentService.RefundPayment"
|
||||
),
|
||||
SagaStep(
|
||||
name="create_shipment",
|
||||
action="ShippingService.CreateShipment",
|
||||
compensation="ShippingService.CancelShipment"
|
||||
),
|
||||
SagaStep(
|
||||
name="send_confirmation",
|
||||
action="NotificationService.SendOrderConfirmation",
|
||||
compensation="NotificationService.SendCancellationNotice"
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
# Usage
|
||||
async def create_order(order_data: Dict):
|
||||
saga = OrderFulfillmentSaga(saga_store, event_publisher)
|
||||
return await saga.start({
|
||||
"order_id": order_data["order_id"],
|
||||
"customer_id": order_data["customer_id"],
|
||||
"items": order_data["items"],
|
||||
"payment_method": order_data["payment_method"],
|
||||
"shipping_address": order_data["shipping_address"]
|
||||
})
|
||||
|
||||
|
||||
# Event handlers in each service
|
||||
class InventoryService:
|
||||
async def handle_reserve_items(self, command: Dict):
|
||||
try:
|
||||
# Reserve inventory
|
||||
reservation = await self.reserve(
|
||||
command["items"],
|
||||
command["order_id"]
|
||||
)
|
||||
# Report success
|
||||
await self.event_publisher.publish(
|
||||
"SagaStepCompleted",
|
||||
{
|
||||
"saga_id": command["saga_id"],
|
||||
"step_name": "reserve_inventory",
|
||||
"result": {"reservation_id": reservation.id}
|
||||
}
|
||||
)
|
||||
except InsufficientInventoryError as e:
|
||||
await self.event_publisher.publish(
|
||||
"SagaStepFailed",
|
||||
{
|
||||
"saga_id": command["saga_id"],
|
||||
"step_name": "reserve_inventory",
|
||||
"error": str(e)
|
||||
}
|
||||
)
|
||||
|
||||
async def handle_release_reservation(self, command: Dict):
|
||||
# Compensating action
|
||||
await self.release_reservation(
|
||||
command["original_result"]["reservation_id"]
|
||||
)
|
||||
await self.event_publisher.publish(
|
||||
"SagaCompensationCompleted",
|
||||
{
|
||||
"saga_id": command["saga_id"],
|
||||
"step_name": "reserve_inventory"
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
### Template 3: Choreography-Based Saga
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Any
|
||||
import asyncio
|
||||
|
||||
@dataclass
|
||||
class SagaContext:
|
||||
"""Passed through choreographed saga events."""
|
||||
saga_id: str
|
||||
step: int
|
||||
data: Dict[str, Any]
|
||||
completed_steps: list
|
||||
|
||||
|
||||
class OrderChoreographySaga:
|
||||
"""Choreography-based saga using events."""
|
||||
|
||||
def __init__(self, event_bus):
|
||||
self.event_bus = event_bus
|
||||
self._register_handlers()
|
||||
|
||||
def _register_handlers(self):
|
||||
self.event_bus.subscribe("OrderCreated", self._on_order_created)
|
||||
self.event_bus.subscribe("InventoryReserved", self._on_inventory_reserved)
|
||||
self.event_bus.subscribe("PaymentProcessed", self._on_payment_processed)
|
||||
self.event_bus.subscribe("ShipmentCreated", self._on_shipment_created)
|
||||
|
||||
# Compensation handlers
|
||||
self.event_bus.subscribe("PaymentFailed", self._on_payment_failed)
|
||||
self.event_bus.subscribe("ShipmentFailed", self._on_shipment_failed)
|
||||
|
||||
async def _on_order_created(self, event: Dict):
|
||||
"""Step 1: Order created, reserve inventory."""
|
||||
await self.event_bus.publish("ReserveInventory", {
|
||||
"saga_id": event["order_id"],
|
||||
"order_id": event["order_id"],
|
||||
"items": event["items"]
|
||||
})
|
||||
|
||||
async def _on_inventory_reserved(self, event: Dict):
|
||||
"""Step 2: Inventory reserved, process payment."""
|
||||
await self.event_bus.publish("ProcessPayment", {
|
||||
"saga_id": event["saga_id"],
|
||||
"order_id": event["order_id"],
|
||||
"amount": event["total_amount"],
|
||||
"reservation_id": event["reservation_id"]
|
||||
})
|
||||
|
||||
async def _on_payment_processed(self, event: Dict):
|
||||
"""Step 3: Payment done, create shipment."""
|
||||
await self.event_bus.publish("CreateShipment", {
|
||||
"saga_id": event["saga_id"],
|
||||
"order_id": event["order_id"],
|
||||
"payment_id": event["payment_id"]
|
||||
})
|
||||
|
||||
async def _on_shipment_created(self, event: Dict):
|
||||
"""Step 4: Complete - send confirmation."""
|
||||
await self.event_bus.publish("OrderFulfilled", {
|
||||
"saga_id": event["saga_id"],
|
||||
"order_id": event["order_id"],
|
||||
"tracking_number": event["tracking_number"]
|
||||
})
|
||||
|
||||
# Compensation handlers
|
||||
async def _on_payment_failed(self, event: Dict):
|
||||
"""Payment failed - release inventory."""
|
||||
await self.event_bus.publish("ReleaseInventory", {
|
||||
"saga_id": event["saga_id"],
|
||||
"reservation_id": event["reservation_id"]
|
||||
})
|
||||
await self.event_bus.publish("OrderFailed", {
|
||||
"order_id": event["order_id"],
|
||||
"reason": "Payment failed"
|
||||
})
|
||||
|
||||
async def _on_shipment_failed(self, event: Dict):
|
||||
"""Shipment failed - refund payment and release inventory."""
|
||||
await self.event_bus.publish("RefundPayment", {
|
||||
"saga_id": event["saga_id"],
|
||||
"payment_id": event["payment_id"]
|
||||
})
|
||||
await self.event_bus.publish("ReleaseInventory", {
|
||||
"saga_id": event["saga_id"],
|
||||
"reservation_id": event["reservation_id"]
|
||||
})
|
||||
```
|
||||
|
||||
### Template 4: Saga with Timeouts
|
||||
|
||||
```python
|
||||
class TimeoutSagaOrchestrator(SagaOrchestrator):
|
||||
"""Saga orchestrator with step timeouts."""
|
||||
|
||||
def __init__(self, saga_store, event_publisher, scheduler):
|
||||
super().__init__(saga_store, event_publisher)
|
||||
self.scheduler = scheduler
|
||||
|
||||
async def _execute_next_step(self, saga: Saga):
|
||||
if saga.current_step >= len(saga.steps):
|
||||
return
|
||||
|
||||
step = saga.steps[saga.current_step]
|
||||
step.status = "executing"
|
||||
step.timeout_at = datetime.utcnow() + timedelta(minutes=5)
|
||||
await self.saga_store.save(saga)
|
||||
|
||||
# Schedule timeout check
|
||||
await self.scheduler.schedule(
|
||||
f"saga_timeout_{saga.saga_id}_{step.name}",
|
||||
self._check_timeout,
|
||||
{"saga_id": saga.saga_id, "step_name": step.name},
|
||||
run_at=step.timeout_at
|
||||
)
|
||||
|
||||
await self.event_publisher.publish(
|
||||
step.action,
|
||||
{"saga_id": saga.saga_id, "step_name": step.name, **saga.data}
|
||||
)
|
||||
|
||||
async def _check_timeout(self, data: Dict):
|
||||
"""Check if step has timed out."""
|
||||
saga = await self.saga_store.get(data["saga_id"])
|
||||
step = next(s for s in saga.steps if s.name == data["step_name"])
|
||||
|
||||
if step.status == "executing":
|
||||
# Step timed out - fail it
|
||||
await self.handle_step_failed(
|
||||
data["saga_id"],
|
||||
data["step_name"],
|
||||
"Step timed out"
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Do's
|
||||
- **Make steps idempotent** - Safe to retry
|
||||
- **Design compensations carefully** - They must work
|
||||
- **Use correlation IDs** - For tracing across services
|
||||
- **Implement timeouts** - Don't wait forever
|
||||
- **Log everything** - For debugging failures
|
||||
|
||||
### Don'ts
|
||||
- **Don't assume instant completion** - Sagas take time
|
||||
- **Don't skip compensation testing** - Most critical part
|
||||
- **Don't couple services** - Use async messaging
|
||||
- **Don't ignore partial failures** - Handle gracefully
|
||||
|
||||
## Resources
|
||||
|
||||
- [Saga Pattern](https://microservices.io/patterns/data/saga.html)
|
||||
- [Designing Data-Intensive Applications](https://dataintensive.net/)
|
||||
Reference in New Issue
Block a user