processors: batch: timeout: 10s send_batch_size: 1024 memory_limiter: check_interval: 5s limit_mib: 512 spike_limit_mib: 128 filter/severity: logs: include: severity_number: min: 9 # INFO 이상만 수집

exporters: awsxray: region: ap-northeast-2 awsemf: region: ap-northeast-2 namespace: AnyCompany/AIOps prometheusremotewrite: endpoint: https://aps-workspaces.ap-northeast-2 .amazonaws.com/workspaces/ws-xxx/api/v1/remote_write auth: authenticator: sigv4auth

# P99 레이턴시 by API endpoint filter ispresent(duration_ms) | stats avg(duration_ms) as avg_ms, pct(duration_ms, 95) as p95, pct(duration_ms, 99) as p99, count(*) as req_count by endpoint | filter req_count > 100 | sort p99 desc

# 로그 그룹별 일일 수집량 (비용 추정) stats sum(strlen(@message)) as total_bytes by @logStream | sort total_bytes desc | limit 20

당신은 AWS 운영 전문가입니다. 다음 인시던트를 분석해 주세요. - 발생 시각: {timestamp} - 영향 서비스: {affected_services} - DevOps Guru Insight: {insight_summary} - 이상 메트릭: {anomaly_metrics} - 최근 변경 사항: {recent_changes} - 관련 로그 (최근 30분): {relevant_logs} 1. 근본 원인 (Root Cause) — 가장 가능성 높은 원인 3가지 2. 인과 관계 체인 — 어떤 순서로 장애가 전파되었는지 3. 영향 범위 — 어떤 서비스/사용자에 영향을 미치는지 4. 즉시 복구 단계 — 우선순위 순서로 5. 재발 방지 — 장기적 개선 방안

import boto3, json bedrock = boto3.client('bedrock-runtime') cw_logs = boto3.client('logs') def handler(event, context): insight = event['detail']['insightDescription'] anomalies = event['detail'].get('anomalies', []) # 관련 로그 수집 logs = collect_recent_logs(anomalies) # Bedrock Claude로 RCA 실행 response = bedrock.invoke_model( modelId='anthropic.claude-sonnet-4-6', body=json.dumps({ 'messages': [{ 'role': 'user', 'content': build_rca_prompt( insight, anomalies, logs) }], 'max_tokens': 2000 }) ) return parse_rca_response(response)

# RCA 결과 → Slack + OpsCenter def send_results(rca_report): # Slack 알림 slack.post_message( channel='#incident', blocks=format_rca_blocks(rca_report) ) # SSM OpsCenter OpsItem 생성 ssm.create_ops_item( Title=f"RCA: {rca_report['root_cause']}", Description=rca_report['full_report'], Severity='2', Source='DevOpsGuru-Bedrock-RCA' ) # 자동 복구 트리거 (Block 3에서 상세) if rca_report['confidence'] > 0.85: trigger_auto_remediation( rca_report['remediation_steps'])

# SSM Automation Runbook: ECS Auto-Scale schemaVersion: '0.3' description: 'ECS Task 자동 스케일링 복구' assumeRole: '{{AutomationAssumeRole}}' parameters: ClusterName: type: String ServiceName: type: String DesiredCount: type: Integer default: 10 mainSteps: - name: checkCurrentState action: aws:executeAwsApi inputs: Service: ecs Api: DescribeServices cluster: '{{ClusterName}}' services: ['{{ServiceName}}'] - name: scaleUp action: aws:executeAwsApi inputs: Service: ecs Api: UpdateService cluster: '{{ClusterName}}' service: '{{ServiceName}}' desiredCount: '{{DesiredCount}}' - name: waitForStable action: aws:waitForAwsResourceProperty inputs: Service: ecs Api: DescribeServices cluster: '{{ClusterName}}' services: ['{{ServiceName}}'] PropertySelector: '$.services[0].runningCount' DesiredValues: ['{{DesiredCount}}'] timeoutSeconds: 300 - name: validateMetrics action: aws:executeScript inputs: Runtime: python3.11 Script: | import boto3 cw = boto3.client('cloudwatch') # 5XX 에러율이 1% 미만인지 확인 # 확인 실패 시 롤백 트리거

# Safety Guard 구현 패턴 class RemediationGuard: def __init__(self, config): self.max_blast_radius = config['max_affected_pct'] self.require_approval = config['require_approval'] self.rollback_conditions = config['rollback_on'] def can_execute(self, action, context): # 1. 영향 범위 확인 affected = self.estimate_blast_radius(action) if affected > self.max_blast_radius: return False, f"Blast radius {affected}% > {self.max_blast_radius}%" # 2. 동시 실행 제한 if self.has_active_remediation(context): return False, "Another remediation in progress" # 3. 승인 필요 여부 if self.require_approval: return 'pending_approval', "Waiting for operator approval" return True, "Safe to execute" def should_rollback(self, metrics_after): for condition in self.rollback_conditions: if self.evaluate(condition, metrics_after): return True, condition['description'] return False, None