monitoring-observability

Monitoring & Observability Core Workflow: Observability Implementation Use this decision tree to determine your starting point: --- 1. Design Metrics Strategy Start with The Four Golden Signals Every service should monitor: Latency (p50/p95/p99), Traffic (req/s), Errors (failure rate), Saturation (resource utilization). RED Method (request-driven): Rate, Errors, Duration | USE Method (infrastructure): Utilization, Saturation, Errors Quick Start - Web Application Example : Deep dive : — metric types, cardinality, naming conventions, dashboard design Querying Metrics Directly Query Prometheus a…

, alert_name):\n issues.append(f\"Alert name '{alert_name}' should use PascalCase (e.g., HighCPUUsage)\")\n\n # Should be descriptive\n if len(alert_name) \u003c 5:\n issues.append(f\"Alert name '{alert_name}' is too short, use descriptive names\")\n\n # Avoid generic names\n generic_names = ['Alert', 'Test', 'Warning', 'Error']\n if alert_name in generic_names:\n issues.append(f\"Alert name '{alert_name}' is too generic\")\n\n return issues\n\n def check_labels(self, alert: Dict[str, Any]) -> List[str]:\n \"\"\"Check required and recommended labels.\"\"\"\n issues = []\n labels = alert.get('labels', {})\n\n # Required labels\n if 'severity' not in labels:\n issues.append(\"Missing required 'severity' label (critical/warning/info)\")\n elif labels['severity'] not in ['critical', 'warning', 'info']:\n issues.append(f\"Severity '{labels['severity']}' should be one of: critical, warning, info\")\n\n # Recommended labels\n if 'team' not in labels:\n self.recommendations.append(\"Consider adding 'team' label for routing\")\n\n if 'component' not in labels and 'service' not in labels:\n self.recommendations.append(\"Consider adding 'component' or 'service' label\")\n\n return issues\n\n def check_annotations(self, alert: Dict[str, Any]) -> List[str]:\n \"\"\"Check annotations quality.\"\"\"\n issues = []\n annotations = alert.get('annotations', {})\n\n # Required annotations\n if 'summary' not in annotations:\n issues.append(\"Missing 'summary' annotation\")\n elif len(annotations['summary']) \u003c 10:\n issues.append(\"Summary annotation is too short, provide clear description\")\n\n if 'description' not in annotations:\n issues.append(\"Missing 'description' annotation\")\n\n # Runbook\n if 'runbook_url' not in annotations and 'runbook' not in annotations:\n self.recommendations.append(\"Consider adding 'runbook_url' for incident response\")\n\n # Check for templating\n if 'summary' in annotations:\n if '{{ $value }}' not in annotations['summary'] and '{{' not in annotations['summary']:\n self.recommendations.append(\"Consider using template variables in summary (e.g., {{ $value }})\")\n\n return issues\n\n def check_expression(self, expr: str, alert_name: str) -> List[str]:\n \"\"\"Check PromQL expression quality.\"\"\"\n issues = []\n\n # Should have a threshold\n if '>' not in expr and '\u003c' not in expr and '==' not in expr and '!=' not in expr:\n issues.append(\"Expression should include a comparison operator\")\n\n # Should use rate() for counters\n if '_total' in expr and 'rate(' not in expr and 'increase(' not in expr:\n self.recommendations.append(\"Consider using rate() or increase() for counter metrics (*_total)\")\n\n # Avoid instant queries without aggregation\n if not any(agg in expr for agg in ['sum(', 'avg(', 'min(', 'max(', 'count(']):\n if expr.count('{') > 1: # Multiple metrics without aggregation\n self.recommendations.append(\"Consider aggregating metrics with sum(), avg(), etc.\")\n\n # Check for proper time windows\n if '[' not in expr and 'rate(' in expr:\n issues.append(\"rate() requires a time window (e.g., rate(metric[5m]))\")\n\n return issues\n\n def check_for_duration(self, rule: Dict[str, Any]) -> List[str]:\n \"\"\"Check for 'for' clause to prevent flapping.\"\"\"\n issues = []\n severity = rule.get('labels', {}).get('severity', 'unknown')\n\n if 'for' not in rule:\n if severity == 'critical':\n issues.append(\"Critical alerts should have 'for' clause to prevent flapping\")\n else:\n self.warnings.append(\"Consider adding 'for' clause to prevent alert flapping\")\n else:\n # Parse duration\n duration = rule['for']\n if severity == 'critical' and any(x in duration for x in ['0s', '30s', '1m']):\n self.warnings.append(f\"'for' duration ({duration}) might be too short for critical alerts\")\n\n return issues\n\n def check_alert_rule(self, rule: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"Check a single alert rule.\"\"\"\n alert_name = rule.get('alert', 'Unknown')\n issues = []\n\n # Check alert name\n issues.extend(self.check_alert_name(alert_name))\n\n # Check expression\n if 'expr' not in rule:\n issues.append(\"Missing 'expr' field\")\n else:\n issues.extend(self.check_expression(rule['expr'], alert_name))\n\n # Check labels\n issues.extend(self.check_labels(rule))\n\n # Check annotations\n issues.extend(self.check_annotations(rule))\n\n # Check for duration\n issues.extend(self.check_for_duration(rule))\n\n return {\n \"alert\": alert_name,\n \"issues\": issues,\n \"severity\": rule.get('labels', {}).get('severity', 'unknown')\n }\n\n def analyze_file(self, filepath: str) -> Dict[str, Any]:\n \"\"\"Analyze a Prometheus rules file.\"\"\"\n try:\n with open(filepath, 'r') as f:\n data = yaml.safe_load(f)\n\n if not data:\n return {\"error\": \"Empty or invalid YAML file\"}\n\n results = []\n groups = data.get('groups', [])\n\n for group in groups:\n group_name = group.get('name', 'Unknown')\n rules = group.get('rules', [])\n\n for rule in rules:\n # Only check alerting rules, not recording rules\n if 'alert' in rule:\n result = self.check_alert_rule(rule)\n result['group'] = group_name\n results.append(result)\n\n return {\n \"file\": filepath,\n \"groups\": len(groups),\n \"alerts_checked\": len(results),\n \"results\": results\n }\n\n except Exception as e:\n return {\"error\": f\"Failed to parse file: {e}\"}\n\n\ndef print_results(analysis: Dict[str, Any], checker: AlertQualityChecker):\n \"\"\"Pretty print analysis results.\"\"\"\n print(\"\\n\" + \"=\"*60)\n print(\"🚨 ALERT QUALITY CHECK RESULTS\")\n print(\"=\"*60)\n\n if \"error\" in analysis:\n print(f\"\\n❌ Error: {analysis['error']}\")\n return\n\n print(f\"\\n📁 File: {analysis['file']}\")\n print(f\"📊 Groups: {analysis['groups']}\")\n print(f\"🔔 Alerts Checked: {analysis['alerts_checked']}\")\n\n # Count issues by severity\n critical_count = 0\n warning_count = 0\n\n for result in analysis['results']:\n if result['issues']:\n critical_count += 1\n\n print(f\"\\n{'='*60}\")\n print(f\"📈 Summary:\")\n print(f\" ❌ Alerts with Issues: {critical_count}\")\n print(f\" ⚠️ Warnings: {len(checker.warnings)}\")\n print(f\" 💡 Recommendations: {len(checker.recommendations)}\")\n\n # Print detailed results\n if critical_count > 0:\n print(f\"\\n{'='*60}\")\n print(\"❌ ALERTS WITH ISSUES:\")\n print(f\"{'='*60}\")\n\n for result in analysis['results']:\n if result['issues']:\n print(f\"\\n🔔 Alert: {result['alert']} (Group: {result['group']})\")\n print(f\" Severity: {result['severity']}\")\n print(\" Issues:\")\n for issue in result['issues']:\n print(f\" • {issue}\")\n\n # Print warnings\n if checker.warnings:\n print(f\"\\n{'='*60}\")\n print(\"⚠️ WARNINGS:\")\n print(f\"{'='*60}\")\n for warning in set(checker.warnings): # Remove duplicates\n print(f\"• {warning}\")\n\n # Print recommendations\n if checker.recommendations:\n print(f\"\\n{'='*60}\")\n print(\"💡 RECOMMENDATIONS:\")\n print(f\"{'='*60}\")\n for rec in list(set(checker.recommendations))[:10]: # Top 10 unique recommendations\n print(f\"• {rec}\")\n\n # Overall score\n total_alerts = analysis['alerts_checked']\n if total_alerts > 0:\n quality_score = ((total_alerts - critical_count) / total_alerts) * 100\n print(f\"\\n{'='*60}\")\n print(f\"📊 Quality Score: {quality_score:.1f}% ({total_alerts - critical_count}/{total_alerts} alerts passing)\")\n print(f\"{'='*60}\\n\")\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Audit Prometheus alert rules for quality and best practices\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Check a single file\n python3 alert_quality_checker.py alerts.yml\n\n # Check all YAML files in a directory\n python3 alert_quality_checker.py /path/to/prometheus/rules/\n\nBest Practices Checked:\n ✓ Alert naming conventions (PascalCase, descriptive)\n ✓ Required labels (severity)\n ✓ Required annotations (summary, description)\n ✓ Runbook URL presence\n ✓ PromQL expression quality\n ✓ 'for' clause to prevent flapping\n ✓ Template variable usage\n \"\"\"\n )\n\n parser.add_argument('path', help='Path to alert rules file or directory')\n parser.add_argument('--verbose', action='store_true', help='Show all recommendations')\n\n args = parser.parse_args()\n\n checker = AlertQualityChecker()\n\n # Check if path is file or directory\n path = Path(args.path)\n\n if path.is_file():\n files = [str(path)]\n elif path.is_dir():\n files = [str(f) for f in path.rglob('*.yml')] + [str(f) for f in path.rglob('*.yaml')]\n else:\n print(f\"❌ Path not found: {args.path}\")\n sys.exit(1)\n\n if not files:\n print(f\"❌ No YAML files found in: {args.path}\")\n sys.exit(1)\n\n print(f\"🔍 Checking {len(files)} file(s)...\")\n\n for filepath in files:\n analysis = checker.analyze_file(filepath)\n print_results(analysis, checker)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":10839,"content_sha256":"b561cf9c41e2de8d5f09557c018110553047d0ad54629bdc7a07a654d76263d1"},{"filename":"scripts/dashboard_generator.py","content":"#!/usr/bin/env python3\n\"\"\"\nGenerate Grafana dashboards from templates.\nSupports: web applications, Kubernetes, databases, Redis, and custom metrics.\n\"\"\"\n\nimport argparse\nimport sys\nimport json\nfrom typing import Dict, List, Any, Optional\nfrom pathlib import Path\n\n\nclass DashboardGenerator:\n def __init__(self, title: str, datasource: str = \"Prometheus\"):\n self.title = title\n self.datasource = datasource\n self.dashboard = self._create_base_dashboard()\n self.panel_id = 1\n self.row_y = 0\n\n def _create_base_dashboard(self) -> Dict[str, Any]:\n \"\"\"Create base dashboard structure.\"\"\"\n return {\n \"dashboard\": {\n \"title\": self.title,\n \"tags\": [],\n \"timezone\": \"browser\",\n \"schemaVersion\": 16,\n \"version\": 0,\n \"refresh\": \"30s\",\n \"panels\": [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n }\n },\n \"overwrite\": True\n }\n\n def add_variable(self, name: str, label: str, query: str):\n \"\"\"Add a template variable.\"\"\"\n variable = {\n \"name\": name,\n \"label\": label,\n \"type\": \"query\",\n \"datasource\": self.datasource,\n \"query\": query,\n \"refresh\": 1,\n \"regex\": \"\",\n \"multi\": False,\n \"includeAll\": False\n }\n self.dashboard[\"dashboard\"][\"templating\"][\"list\"].append(variable)\n\n def add_row(self, title: str):\n \"\"\"Add a row panel.\"\"\"\n panel = {\n \"id\": self.panel_id,\n \"type\": \"row\",\n \"title\": title,\n \"collapsed\": False,\n \"gridPos\": {\"h\": 1, \"w\": 24, \"x\": 0, \"y\": self.row_y}\n }\n self.dashboard[\"dashboard\"][\"panels\"].append(panel)\n self.panel_id += 1\n self.row_y += 1\n\n def add_graph(self, title: str, targets: List[Dict[str, str]], unit: str = \"short\",\n width: int = 12, height: int = 8):\n \"\"\"Add a graph panel.\"\"\"\n panel = {\n \"id\": self.panel_id,\n \"type\": \"graph\",\n \"title\": title,\n \"datasource\": self.datasource,\n \"targets\": [\n {\n \"expr\": target[\"query\"],\n \"legendFormat\": target.get(\"legend\", \"\"),\n \"refId\": chr(65 + i) # A, B, C, etc.\n }\n for i, target in enumerate(targets)\n ],\n \"gridPos\": {\"h\": height, \"w\": width, \"x\": 0, \"y\": self.row_y},\n \"yaxes\": [\n {\"format\": unit, \"label\": None, \"show\": True},\n {\"format\": \"short\", \"label\": None, \"show\": True}\n ],\n \"lines\": True,\n \"fill\": 1,\n \"linewidth\": 2,\n \"legend\": {\n \"show\": True,\n \"alignAsTable\": True,\n \"avg\": True,\n \"current\": True,\n \"max\": True,\n \"min\": False,\n \"total\": False,\n \"values\": True\n }\n }\n self.dashboard[\"dashboard\"][\"panels\"].append(panel)\n self.panel_id += 1\n self.row_y += height\n\n def add_stat(self, title: str, query: str, unit: str = \"short\",\n width: int = 6, height: int = 4):\n \"\"\"Add a stat panel (single value).\"\"\"\n panel = {\n \"id\": self.panel_id,\n \"type\": \"stat\",\n \"title\": title,\n \"datasource\": self.datasource,\n \"targets\": [\n {\n \"expr\": query,\n \"refId\": \"A\"\n }\n ],\n \"gridPos\": {\"h\": height, \"w\": width, \"x\": 0, \"y\": self.row_y},\n \"options\": {\n \"graphMode\": \"area\",\n \"orientation\": \"auto\",\n \"reduceOptions\": {\n \"values\": False,\n \"calcs\": [\"lastNotNull\"]\n }\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": unit,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\"value\": None, \"color\": \"green\"},\n {\"value\": 80, \"color\": \"red\"}\n ]\n }\n }\n }\n }\n self.dashboard[\"dashboard\"][\"panels\"].append(panel)\n self.panel_id += 1\n\n def generate_webapp_dashboard(self, service: str):\n \"\"\"Generate dashboard for web application.\"\"\"\n self.add_variable(\"service\", \"Service\", f\"label_values({service}_http_requests_total, service)\")\n\n # Request metrics\n self.add_row(\"Request Metrics\")\n\n self.add_graph(\n \"Request Rate\",\n [{\"query\": f'sum(rate({service}_http_requests_total[5m])) by (status)', \"legend\": \"{{status}}\"}],\n unit=\"reqps\",\n width=12\n )\n\n self.add_graph(\n \"Request Latency (p50, p95, p99)\",\n [\n {\"query\": f'histogram_quantile(0.50, sum(rate({service}_http_request_duration_seconds_bucket[5m])) by (le))', \"legend\": \"p50\"},\n {\"query\": f'histogram_quantile(0.95, sum(rate({service}_http_request_duration_seconds_bucket[5m])) by (le))', \"legend\": \"p95\"},\n {\"query\": f'histogram_quantile(0.99, sum(rate({service}_http_request_duration_seconds_bucket[5m])) by (le))', \"legend\": \"p99\"}\n ],\n unit=\"s\",\n width=12\n )\n\n # Error rate\n self.add_row(\"Errors\")\n\n self.add_graph(\n \"Error Rate (%)\",\n [{\"query\": f'sum(rate({service}_http_requests_total{{status=~\"5..\"}}[5m])) / sum(rate({service}_http_requests_total[5m])) * 100', \"legend\": \"Error Rate\"}],\n unit=\"percent\",\n width=12\n )\n\n # Resource usage\n self.add_row(\"Resource Usage\")\n\n self.add_graph(\n \"CPU Usage\",\n [{\"query\": f'sum(rate(process_cpu_seconds_total{{job=\"{service}\"}}[5m])) * 100', \"legend\": \"CPU %\"}],\n unit=\"percent\",\n width=12\n )\n\n self.add_graph(\n \"Memory Usage\",\n [{\"query\": f'process_resident_memory_bytes{{job=\"{service}\"}}', \"legend\": \"Memory\"}],\n unit=\"bytes\",\n width=12\n )\n\n def generate_kubernetes_dashboard(self, namespace: str):\n \"\"\"Generate dashboard for Kubernetes cluster.\"\"\"\n self.add_variable(\"namespace\", \"Namespace\", f\"label_values(kube_pod_info, namespace)\")\n\n # Cluster overview\n self.add_row(\"Cluster Overview\")\n\n self.add_stat(\"Total Pods\", f'count(kube_pod_info{{namespace=\"{namespace}\"}})', width=6)\n self.add_stat(\"Running Pods\", f'count(kube_pod_status_phase{{namespace=\"{namespace}\", phase=\"Running\"}})', width=6)\n self.add_stat(\"Pending Pods\", f'count(kube_pod_status_phase{{namespace=\"{namespace}\", phase=\"Pending\"}})', width=6)\n self.add_stat(\"Failed Pods\", f'count(kube_pod_status_phase{{namespace=\"{namespace}\", phase=\"Failed\"}})', width=6)\n\n # Resource usage\n self.add_row(\"Resource Usage\")\n\n self.add_graph(\n \"CPU Usage by Pod\",\n [{\"query\": f'sum(rate(container_cpu_usage_seconds_total{{namespace=\"{namespace}\"}}[5m])) by (pod)', \"legend\": \"{{pod}}\"}],\n unit=\"percent\",\n width=12\n )\n\n self.add_graph(\n \"Memory Usage by Pod\",\n [{\"query\": f'sum(container_memory_usage_bytes{{namespace=\"{namespace}\"}}) by (pod)', \"legend\": \"{{pod}}\"}],\n unit=\"bytes\",\n width=12\n )\n\n # Network\n self.add_row(\"Network\")\n\n self.add_graph(\n \"Network I/O\",\n [\n {\"query\": f'sum(rate(container_network_receive_bytes_total{{namespace=\"{namespace}\"}}[5m])) by (pod)', \"legend\": \"Receive - {{pod}}\"},\n {\"query\": f'sum(rate(container_network_transmit_bytes_total{{namespace=\"{namespace}\"}}[5m])) by (pod)', \"legend\": \"Transmit - {{pod}}\"}\n ],\n unit=\"Bps\",\n width=12\n )\n\n def generate_database_dashboard(self, db_type: str, instance: str):\n \"\"\"Generate dashboard for database (postgres/mysql).\"\"\"\n if db_type == \"postgres\":\n self._generate_postgres_dashboard(instance)\n elif db_type == \"mysql\":\n self._generate_mysql_dashboard(instance)\n\n def _generate_postgres_dashboard(self, instance: str):\n \"\"\"Generate PostgreSQL dashboard.\"\"\"\n self.add_row(\"PostgreSQL Metrics\")\n\n self.add_graph(\n \"Connections\",\n [\n {\"query\": f'pg_stat_database_numbackends{{instance=\"{instance}\"}}', \"legend\": \"{{datname}}\"}\n ],\n unit=\"short\",\n width=12\n )\n\n self.add_graph(\n \"Transactions per Second\",\n [\n {\"query\": f'rate(pg_stat_database_xact_commit{{instance=\"{instance}\"}}[5m])', \"legend\": \"Commits\"},\n {\"query\": f'rate(pg_stat_database_xact_rollback{{instance=\"{instance}\"}}[5m])', \"legend\": \"Rollbacks\"}\n ],\n unit=\"tps\",\n width=12\n )\n\n self.add_graph(\n \"Query Duration (p95)\",\n [\n {\"query\": f'histogram_quantile(0.95, rate(pg_stat_statements_total_time_bucket{{instance=\"{instance}\"}}[5m]))', \"legend\": \"p95\"}\n ],\n unit=\"ms\",\n width=12\n )\n\n def _generate_mysql_dashboard(self, instance: str):\n \"\"\"Generate MySQL dashboard.\"\"\"\n self.add_row(\"MySQL Metrics\")\n\n self.add_graph(\n \"Connections\",\n [\n {\"query\": f'mysql_global_status_threads_connected{{instance=\"{instance}\"}}', \"legend\": \"Connected\"},\n {\"query\": f'mysql_global_status_threads_running{{instance=\"{instance}\"}}', \"legend\": \"Running\"}\n ],\n unit=\"short\",\n width=12\n )\n\n self.add_graph(\n \"Queries per Second\",\n [\n {\"query\": f'rate(mysql_global_status_queries{{instance=\"{instance}\"}}[5m])', \"legend\": \"Queries\"}\n ],\n unit=\"qps\",\n width=12\n )\n\n def save(self, output_file: str):\n \"\"\"Save dashboard to file.\"\"\"\n try:\n with open(output_file, 'w') as f:\n json.dump(self.dashboard, f, indent=2)\n return True\n except Exception as e:\n print(f\"❌ Error saving dashboard: {e}\")\n return False\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Generate Grafana dashboards from templates\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Web application dashboard\n python3 dashboard_generator.py webapp \\\\\n --title \"My API Dashboard\" \\\\\n --service my_api \\\\\n --output dashboard.json\n\n # Kubernetes dashboard\n python3 dashboard_generator.py kubernetes \\\\\n --title \"K8s Namespace\" \\\\\n --namespace production \\\\\n --output k8s-dashboard.json\n\n # Database dashboard\n python3 dashboard_generator.py database \\\\\n --title \"PostgreSQL\" \\\\\n --db-type postgres \\\\\n --instance db.example.com:5432 \\\\\n --output db-dashboard.json\n \"\"\"\n )\n\n parser.add_argument('type', choices=['webapp', 'kubernetes', 'database'],\n help='Dashboard type')\n parser.add_argument('--title', required=True, help='Dashboard title')\n parser.add_argument('--output', required=True, help='Output file path')\n parser.add_argument('--datasource', default='Prometheus', help='Data source name')\n\n # Web app specific\n parser.add_argument('--service', help='Service name (for webapp)')\n\n # Kubernetes specific\n parser.add_argument('--namespace', help='Kubernetes namespace')\n\n # Database specific\n parser.add_argument('--db-type', choices=['postgres', 'mysql'], help='Database type')\n parser.add_argument('--instance', help='Database instance')\n\n args = parser.parse_args()\n\n print(f\"🎨 Generating {args.type} dashboard: {args.title}\")\n\n generator = DashboardGenerator(args.title, args.datasource)\n\n if args.type == 'webapp':\n if not args.service:\n print(\"❌ --service required for webapp dashboard\")\n sys.exit(1)\n generator.generate_webapp_dashboard(args.service)\n\n elif args.type == 'kubernetes':\n if not args.namespace:\n print(\"❌ --namespace required for kubernetes dashboard\")\n sys.exit(1)\n generator.generate_kubernetes_dashboard(args.namespace)\n\n elif args.type == 'database':\n if not args.db_type or not args.instance:\n print(\"❌ --db-type and --instance required for database dashboard\")\n sys.exit(1)\n generator.generate_database_dashboard(args.db_type, args.instance)\n\n if generator.save(args.output):\n print(f\"✅ Dashboard saved to: {args.output}\")\n print(f\"\\n📝 Import to Grafana:\")\n print(f\" 1. Go to Grafana → Dashboards → Import\")\n print(f\" 2. Upload {args.output}\")\n print(f\" 3. Select datasource and save\")\n else:\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":13556,"content_sha256":"6fe98a49ae431d67bc44eb631c542ba29199da72cc348e90ec99d73a05783ee5"},{"filename":"scripts/log_analyzer.py","content":"#!/usr/bin/env python3\n\"\"\"\nParse and analyze logs for patterns, errors, and anomalies.\nSupports: error detection, frequency analysis, pattern matching.\n\"\"\"\n\nimport argparse\nimport sys\nimport re\nimport json\nfrom collections import Counter, defaultdict\nfrom datetime import datetime\nfrom typing import Dict, List, Any, Optional\nfrom pathlib import Path\n\ntry:\n from tabulate import tabulate\nexcept ImportError:\n tabulate = None\n\n\nclass LogAnalyzer:\n # Common log level patterns\n LOG_LEVELS = {\n 'ERROR': r'\\b(ERROR|Error|error)\\b',\n 'WARN': r'\\b(WARN|Warning|warn|warning)\\b',\n 'INFO': r'\\b(INFO|Info|info)\\b',\n 'DEBUG': r'\\b(DEBUG|Debug|debug)\\b',\n 'FATAL': r'\\b(FATAL|Fatal|fatal|CRITICAL|Critical)\\b'\n }\n\n # Common error patterns\n ERROR_PATTERNS = {\n 'exception': r'Exception|exception|EXCEPTION',\n 'stack_trace': r'\\s+at\\s+.*\$.*:\\d+\$',\n 'http_error': r'\\b[45]\\d{2}\\b', # 4xx and 5xx HTTP codes\n 'timeout': r'timeout|timed out|TIMEOUT',\n 'connection_refused': r'connection refused|ECONNREFUSED',\n 'out_of_memory': r'OutOfMemoryError|OOM|out of memory',\n 'null_pointer': r'NullPointerException|null pointer|NPE',\n 'database_error': r'SQLException|database error|DB error'\n }\n\n def __init__(self, log_file: str):\n self.log_file = log_file\n self.lines = []\n self.log_levels = Counter()\n self.error_patterns = Counter()\n self.timestamps = []\n\n def parse_file(self) -> bool:\n \"\"\"Parse log file.\"\"\"\n try:\n with open(self.log_file, 'r', encoding='utf-8', errors='ignore') as f:\n self.lines = f.readlines()\n return True\n except Exception as e:\n print(f\"❌ Error reading file: {e}\")\n return False\n\n def analyze_log_levels(self):\n \"\"\"Count log levels.\"\"\"\n for line in self.lines:\n for level, pattern in self.LOG_LEVELS.items():\n if re.search(pattern, line):\n self.log_levels[level] += 1\n break # Count each line only once\n\n def analyze_error_patterns(self):\n \"\"\"Detect common error patterns.\"\"\"\n for line in self.lines:\n for pattern_name, pattern in self.ERROR_PATTERNS.items():\n if re.search(pattern, line, re.IGNORECASE):\n self.error_patterns[pattern_name] += 1\n\n def extract_timestamps(self, timestamp_pattern: Optional[str] = None):\n \"\"\"Extract timestamps from logs.\"\"\"\n if not timestamp_pattern:\n # Common timestamp patterns\n patterns = [\n r'\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}', # ISO format\n r'\\d{2}/\\w{3}/\\d{4}:\\d{2}:\\d{2}:\\d{2}', # Apache format\n r'\\w{3}\\s+\\d{1,2}\\s+\\d{2}:\\d{2}:\\d{2}', # Syslog format\n ]\n else:\n patterns = [timestamp_pattern]\n\n for line in self.lines:\n for pattern in patterns:\n match = re.search(pattern, line)\n if match:\n self.timestamps.append(match.group())\n break\n\n def find_error_lines(self, context: int = 2) -> List[Dict[str, Any]]:\n \"\"\"Find error lines with context.\"\"\"\n errors = []\n\n for i, line in enumerate(self.lines):\n # Check if line contains error keywords\n is_error = any(re.search(pattern, line, re.IGNORECASE)\n for pattern in [self.LOG_LEVELS['ERROR'], self.LOG_LEVELS['FATAL']])\n\n if is_error:\n # Get context lines\n start = max(0, i - context)\n end = min(len(self.lines), i + context + 1)\n context_lines = self.lines[start:end]\n\n errors.append({\n 'line_number': i + 1,\n 'line': line.strip(),\n 'context': ''.join(context_lines)\n })\n\n return errors\n\n def analyze_frequency(self, time_window_minutes: int = 5) -> Dict[str, Any]:\n \"\"\"Analyze log frequency over time.\"\"\"\n if not self.timestamps:\n return {\"error\": \"No timestamps found\"}\n\n # This is a simplified version - in production you'd parse actual timestamps\n total_lines = len(self.lines)\n if self.timestamps:\n time_span = len(self.timestamps)\n avg_per_window = total_lines / max(1, time_span / time_window_minutes)\n else:\n avg_per_window = 0\n\n return {\n \"total_lines\": total_lines,\n \"timestamps_found\": len(self.timestamps),\n \"avg_per_window\": avg_per_window\n }\n\n def extract_unique_messages(self, pattern: str) -> List[str]:\n \"\"\"Extract unique messages matching a pattern.\"\"\"\n matches = []\n seen = set()\n\n for line in self.lines:\n match = re.search(pattern, line, re.IGNORECASE)\n if match:\n msg = match.group() if match.lastindex is None else match.group(1)\n if msg not in seen:\n matches.append(msg)\n seen.add(msg)\n\n return matches\n\n def find_stack_traces(self) -> List[Dict[str, Any]]:\n \"\"\"Extract complete stack traces.\"\"\"\n stack_traces = []\n current_trace = []\n in_trace = False\n\n for i, line in enumerate(self.lines):\n # Start of stack trace\n if re.search(r'Exception|Error.*:', line):\n if current_trace:\n stack_traces.append({\n 'line_start': i - len(current_trace) + 1,\n 'trace': '\\n'.join(current_trace)\n })\n current_trace = [line.strip()]\n in_trace = True\n # Stack trace continuation\n elif in_trace and re.search(r'^\\s+at\\s+', line):\n current_trace.append(line.strip())\n # End of stack trace\n elif in_trace:\n if current_trace:\n stack_traces.append({\n 'line_start': i - len(current_trace) + 1,\n 'trace': '\\n'.join(current_trace)\n })\n current_trace = []\n in_trace = False\n\n # Add last trace if exists\n if current_trace:\n stack_traces.append({\n 'line_start': len(self.lines) - len(current_trace) + 1,\n 'trace': '\\n'.join(current_trace)\n })\n\n return stack_traces\n\n\ndef print_analysis_results(analyzer: LogAnalyzer, show_errors: bool = False,\n show_traces: bool = False):\n \"\"\"Print analysis results.\"\"\"\n print(\"\\n\" + \"=\"*60)\n print(\"📝 LOG ANALYSIS RESULTS\")\n print(\"=\"*60)\n\n print(f\"\\n📁 File: {analyzer.log_file}\")\n print(f\"📊 Total Lines: {len(analyzer.lines):,}\")\n\n # Log levels\n if analyzer.log_levels:\n print(f\"\\n{'='*60}\")\n print(\"📊 LOG LEVEL DISTRIBUTION:\")\n print(f\"{'='*60}\")\n\n level_emoji = {\n 'FATAL': '🔴',\n 'ERROR': '❌',\n 'WARN': '⚠️',\n 'INFO': 'ℹ️',\n 'DEBUG': '🐛'\n }\n\n for level, count in analyzer.log_levels.most_common():\n emoji = level_emoji.get(level, '•')\n percentage = (count / len(analyzer.lines)) * 100\n print(f\"{emoji} {level:10s}: {count:6,} ({percentage:5.1f}%)\")\n\n # Error patterns\n if analyzer.error_patterns:\n print(f\"\\n{'='*60}\")\n print(\"🔍 ERROR PATTERNS DETECTED:\")\n print(f\"{'='*60}\")\n\n for pattern, count in analyzer.error_patterns.most_common(10):\n print(f\"• {pattern:20s}: {count:,} occurrences\")\n\n # Timestamps\n if analyzer.timestamps:\n print(f\"\\n{'='*60}\")\n print(f\"⏰ Timestamps Found: {len(analyzer.timestamps):,}\")\n print(f\" First: {analyzer.timestamps[0]}\")\n print(f\" Last: {analyzer.timestamps[-1]}\")\n\n # Error lines\n if show_errors:\n errors = analyzer.find_error_lines(context=1)\n if errors:\n print(f\"\\n{'='*60}\")\n print(f\"❌ ERROR LINES (showing first 10 of {len(errors)}):\")\n print(f\"{'='*60}\")\n\n for error in errors[:10]:\n print(f\"\\nLine {error['line_number']}:\")\n print(f\" {error['line']}\")\n\n # Stack traces\n if show_traces:\n traces = analyzer.find_stack_traces()\n if traces:\n print(f\"\\n{'='*60}\")\n print(f\"📚 STACK TRACES FOUND: {len(traces)}\")\n print(f\"{'='*60}\")\n\n for i, trace in enumerate(traces[:5], 1):\n print(f\"\\nTrace {i} (starting at line {trace['line_start']}):\")\n print(trace['trace'])\n if i \u003c len(traces):\n print(\"\\n\" + \"-\"*60)\n\n print(\"\\n\" + \"=\"*60)\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Analyze log files for errors, patterns, and anomalies\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Basic analysis\n python3 log_analyzer.py application.log\n\n # Show error lines with context\n python3 log_analyzer.py application.log --show-errors\n\n # Show stack traces\n python3 log_analyzer.py application.log --show-traces\n\n # Full analysis\n python3 log_analyzer.py application.log --show-errors --show-traces\n\nFeatures:\n • Log level distribution (ERROR, WARN, INFO, DEBUG, FATAL)\n • Common error pattern detection\n • Timestamp extraction\n • Error line identification with context\n • Stack trace extraction\n • Frequency analysis\n \"\"\"\n )\n\n parser.add_argument('log_file', help='Path to log file')\n parser.add_argument('--show-errors', action='store_true', help='Show error lines')\n parser.add_argument('--show-traces', action='store_true', help='Show stack traces')\n parser.add_argument('--timestamp-pattern', help='Custom regex for timestamp extraction')\n\n args = parser.parse_args()\n\n if not Path(args.log_file).exists():\n print(f\"❌ File not found: {args.log_file}\")\n sys.exit(1)\n\n print(f\"🔍 Analyzing log file: {args.log_file}\")\n\n analyzer = LogAnalyzer(args.log_file)\n\n if not analyzer.parse_file():\n sys.exit(1)\n\n # Perform analysis\n analyzer.analyze_log_levels()\n analyzer.analyze_error_patterns()\n analyzer.extract_timestamps(args.timestamp_pattern)\n\n # Print results\n print_analysis_results(analyzer, args.show_errors, args.show_traces)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":10683,"content_sha256":"c7fb7e13c2d6507c81ee9575fc8514408d36b2f2e786caeb536ba927d517046e"},{"filename":"scripts/slo_calculator.py","content":"#!/usr/bin/env python3\n\"\"\"\nCalculate SLO compliance, error budgets, and burn rates.\nSupports availability SLOs and latency SLOs.\n\"\"\"\n\nimport argparse\nimport sys\nfrom datetime import datetime, timedelta\nfrom typing import Dict, Any, Optional\n\ntry:\n from tabulate import tabulate\nexcept ImportError:\n print(\"⚠️ Warning: 'tabulate' library not found. Install with: pip install tabulate\")\n tabulate = None\n\n\nclass SLOCalculator:\n # SLO targets and allowed downtime per period\n SLO_TARGETS = {\n \"90.0\": {\"year\": 36.5, \"month\": 3.0, \"week\": 0.7, \"day\": 0.1}, # days\n \"95.0\": {\"year\": 18.25, \"month\": 1.5, \"week\": 0.35, \"day\": 0.05},\n \"99.0\": {\"year\": 3.65, \"month\": 0.3, \"week\": 0.07, \"day\": 0.01},\n \"99.5\": {\"year\": 1.83, \"month\": 0.15, \"week\": 0.035, \"day\": 0.005},\n \"99.9\": {\"year\": 0.365, \"month\": 0.03, \"week\": 0.007, \"day\": 0.001},\n \"99.95\": {\"year\": 0.183, \"month\": 0.015, \"week\": 0.0035, \"day\": 0.0005},\n \"99.99\": {\"year\": 0.0365, \"month\": 0.003, \"week\": 0.0007, \"day\": 0.0001},\n }\n\n def __init__(self, slo_target: float, period_days: int = 30):\n \"\"\"\n Initialize SLO calculator.\n\n Args:\n slo_target: SLO target percentage (e.g., 99.9)\n period_days: Time period in days (default: 30)\n \"\"\"\n self.slo_target = slo_target\n self.period_days = period_days\n self.error_budget_minutes = self.calculate_error_budget_minutes()\n\n def calculate_error_budget_minutes(self) -> float:\n \"\"\"Calculate error budget in minutes for the period.\"\"\"\n total_minutes = self.period_days * 24 * 60\n allowed_error_rate = (100 - self.slo_target) / 100\n return total_minutes * allowed_error_rate\n\n def calculate_availability_slo(self, total_requests: int, failed_requests: int) -> Dict[str, Any]:\n \"\"\"\n Calculate availability SLO compliance.\n\n Args:\n total_requests: Total number of requests\n failed_requests: Number of failed requests\n\n Returns:\n Dict with SLO compliance metrics\n \"\"\"\n if total_requests == 0:\n return {\n \"error\": \"No requests in the period\",\n \"slo_met\": False\n }\n\n success_rate = ((total_requests - failed_requests) / total_requests) * 100\n error_rate = (failed_requests / total_requests) * 100\n\n # Calculate error budget consumption\n allowed_failures = total_requests * ((100 - self.slo_target) / 100)\n error_budget_consumed = (failed_requests / allowed_failures) * 100 if allowed_failures > 0 else float('inf')\n error_budget_remaining = max(0, 100 - error_budget_consumed)\n\n # Determine if SLO is met\n slo_met = success_rate >= self.slo_target\n\n return {\n \"slo_target\": self.slo_target,\n \"period_days\": self.period_days,\n \"total_requests\": total_requests,\n \"failed_requests\": failed_requests,\n \"success_requests\": total_requests - failed_requests,\n \"success_rate\": success_rate,\n \"error_rate\": error_rate,\n \"slo_met\": slo_met,\n \"error_budget_total\": allowed_failures,\n \"error_budget_consumed\": error_budget_consumed,\n \"error_budget_remaining\": error_budget_remaining,\n \"margin\": success_rate - self.slo_target\n }\n\n def calculate_latency_slo(self, total_requests: int, requests_exceeding_threshold: int) -> Dict[str, Any]:\n \"\"\"\n Calculate latency SLO compliance.\n\n Args:\n total_requests: Total number of requests\n requests_exceeding_threshold: Number of requests exceeding latency threshold\n\n Returns:\n Dict with SLO compliance metrics\n \"\"\"\n if total_requests == 0:\n return {\n \"error\": \"No requests in the period\",\n \"slo_met\": False\n }\n\n within_threshold_rate = ((total_requests - requests_exceeding_threshold) / total_requests) * 100\n\n # Calculate error budget consumption\n allowed_slow_requests = total_requests * ((100 - self.slo_target) / 100)\n error_budget_consumed = (requests_exceeding_threshold / allowed_slow_requests) * 100 if allowed_slow_requests > 0 else float('inf')\n error_budget_remaining = max(0, 100 - error_budget_consumed)\n\n slo_met = within_threshold_rate >= self.slo_target\n\n return {\n \"slo_target\": self.slo_target,\n \"period_days\": self.period_days,\n \"total_requests\": total_requests,\n \"requests_exceeding_threshold\": requests_exceeding_threshold,\n \"requests_within_threshold\": total_requests - requests_exceeding_threshold,\n \"within_threshold_rate\": within_threshold_rate,\n \"slo_met\": slo_met,\n \"error_budget_total\": allowed_slow_requests,\n \"error_budget_consumed\": error_budget_consumed,\n \"error_budget_remaining\": error_budget_remaining,\n \"margin\": within_threshold_rate - self.slo_target\n }\n\n def calculate_burn_rate(self, errors_in_window: int, requests_in_window: int, window_hours: float) -> Dict[str, Any]:\n \"\"\"\n Calculate error budget burn rate.\n\n Args:\n errors_in_window: Number of errors in the time window\n requests_in_window: Total requests in the time window\n window_hours: Size of the time window in hours\n\n Returns:\n Dict with burn rate metrics\n \"\"\"\n if requests_in_window == 0:\n return {\"error\": \"No requests in window\"}\n\n # Calculate actual error rate in this window\n actual_error_rate = (errors_in_window / requests_in_window) * 100\n\n # Calculate allowed error rate for SLO\n allowed_error_rate = 100 - self.slo_target\n\n # Burn rate = actual error rate / allowed error rate\n burn_rate = actual_error_rate / allowed_error_rate if allowed_error_rate > 0 else float('inf')\n\n # Calculate time to exhaustion\n if burn_rate > 0:\n error_budget_hours = self.error_budget_minutes / 60\n hours_to_exhaustion = error_budget_hours / burn_rate\n else:\n hours_to_exhaustion = float('inf')\n\n # Determine severity\n if burn_rate >= 14.4: # 1 hour window, burns budget in 2 days\n severity = \"critical\"\n elif burn_rate >= 6: # 6 hour window, burns budget in 5 days\n severity = \"warning\"\n elif burn_rate >= 1:\n severity = \"elevated\"\n else:\n severity = \"normal\"\n\n return {\n \"window_hours\": window_hours,\n \"requests_in_window\": requests_in_window,\n \"errors_in_window\": errors_in_window,\n \"actual_error_rate\": actual_error_rate,\n \"allowed_error_rate\": allowed_error_rate,\n \"burn_rate\": burn_rate,\n \"hours_to_exhaustion\": hours_to_exhaustion,\n \"severity\": severity\n }\n\n @staticmethod\n def print_slo_table():\n \"\"\"Print table of common SLO targets and allowed downtime.\"\"\"\n if not tabulate:\n print(\"Install tabulate for formatted output: pip install tabulate\")\n return\n\n print(\"\\n📊 SLO TARGETS AND ALLOWED DOWNTIME\")\n print(\"=\"*60)\n\n headers = [\"SLO\", \"Year\", \"Month\", \"Week\", \"Day\"]\n rows = []\n\n for slo, downtimes in sorted(SLOCalculator.SLO_TARGETS.items(), reverse=True):\n row = [\n f\"{slo}%\",\n f\"{downtimes['year']:.2f} days\",\n f\"{downtimes['month']:.2f} days\",\n f\"{downtimes['week']:.2f} days\",\n f\"{downtimes['day']:.2f} days\"\n ]\n rows.append(row)\n\n print(tabulate(rows, headers=headers, tablefmt=\"grid\"))\n\n\ndef print_availability_results(results: Dict[str, Any]):\n \"\"\"Print availability SLO results.\"\"\"\n print(\"\\n\" + \"=\"*60)\n print(\"📊 AVAILABILITY SLO COMPLIANCE\")\n print(\"=\"*60)\n\n if \"error\" in results:\n print(f\"\\n❌ Error: {results['error']}\")\n return\n\n status_emoji = \"✅\" if results['slo_met'] else \"❌\"\n print(f\"\\n{status_emoji} SLO Status: {'MET' if results['slo_met'] else 'VIOLATED'}\")\n print(f\" Target: {results['slo_target']}%\")\n print(f\" Actual: {results['success_rate']:.3f}%\")\n print(f\" Margin: {results['margin']:+.3f}%\")\n\n print(f\"\\n📈 Request Statistics:\")\n print(f\" Total Requests: {results['total_requests']:,}\")\n print(f\" Successful: {results['success_requests']:,}\")\n print(f\" Failed: {results['failed_requests']:,}\")\n print(f\" Error Rate: {results['error_rate']:.3f}%\")\n\n print(f\"\\n💰 Error Budget:\")\n budget_emoji = \"✅\" if results['error_budget_remaining'] > 20 else \"⚠️\" if results['error_budget_remaining'] > 0 else \"❌\"\n print(f\" {budget_emoji} Remaining: {results['error_budget_remaining']:.1f}%\")\n print(f\" Consumed: {results['error_budget_consumed']:.1f}%\")\n print(f\" Allowed Failures: {results['error_budget_total']:.0f}\")\n\n print(\"\\n\" + \"=\"*60)\n\n\ndef print_burn_rate_results(results: Dict[str, Any]):\n \"\"\"Print burn rate results.\"\"\"\n print(\"\\n\" + \"=\"*60)\n print(\"🔥 ERROR BUDGET BURN RATE\")\n print(\"=\"*60)\n\n if \"error\" in results:\n print(f\"\\n❌ Error: {results['error']}\")\n return\n\n severity_emoji = {\n \"critical\": \"🔴\",\n \"warning\": \"🟡\",\n \"elevated\": \"🟠\",\n \"normal\": \"🟢\"\n }\n\n print(f\"\\n{severity_emoji.get(results['severity'], '❓')} Severity: {results['severity'].upper()}\")\n print(f\" Burn Rate: {results['burn_rate']:.2f}x\")\n print(f\" Time to Exhaustion: {results['hours_to_exhaustion']:.1f} hours ({results['hours_to_exhaustion']/24:.1f} days)\")\n\n print(f\"\\n📊 Window Statistics:\")\n print(f\" Window: {results['window_hours']} hours\")\n print(f\" Requests: {results['requests_in_window']:,}\")\n print(f\" Errors: {results['errors_in_window']:,}\")\n print(f\" Actual Error Rate: {results['actual_error_rate']:.3f}%\")\n print(f\" Allowed Error Rate: {results['allowed_error_rate']:.3f}%\")\n\n print(\"\\n\" + \"=\"*60)\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Calculate SLO compliance and error budgets\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Show SLO reference table\n python3 slo_calculator.py --table\n\n # Calculate availability SLO\n python3 slo_calculator.py availability \\\\\n --slo 99.9 \\\\\n --total-requests 1000000 \\\\\n --failed-requests 1500 \\\\\n --period-days 30\n\n # Calculate latency SLO\n python3 slo_calculator.py latency \\\\\n --slo 99.5 \\\\\n --total-requests 500000 \\\\\n --slow-requests 3000 \\\\\n --period-days 7\n\n # Calculate burn rate\n python3 slo_calculator.py burn-rate \\\\\n --slo 99.9 \\\\\n --errors 50 \\\\\n --requests 10000 \\\\\n --window-hours 1\n \"\"\"\n )\n\n parser.add_argument('mode', nargs='?', choices=['availability', 'latency', 'burn-rate'],\n help='Calculation mode')\n parser.add_argument('--table', action='store_true', help='Show SLO reference table')\n parser.add_argument('--slo', type=float, help='SLO target percentage (e.g., 99.9)')\n parser.add_argument('--period-days', type=int, default=30, help='Period in days (default: 30)')\n\n # Availability SLO arguments\n parser.add_argument('--total-requests', type=int, help='Total number of requests')\n parser.add_argument('--failed-requests', type=int, help='Number of failed requests')\n\n # Latency SLO arguments\n parser.add_argument('--slow-requests', type=int, help='Number of requests exceeding threshold')\n\n # Burn rate arguments\n parser.add_argument('--errors', type=int, help='Number of errors in window')\n parser.add_argument('--requests', type=int, help='Number of requests in window')\n parser.add_argument('--window-hours', type=float, help='Window size in hours')\n\n args = parser.parse_args()\n\n # Show table if requested\n if args.table:\n SLOCalculator.print_slo_table()\n return\n\n if not args.mode:\n parser.print_help()\n return\n\n if not args.slo:\n print(\"❌ --slo required\")\n sys.exit(1)\n\n calculator = SLOCalculator(args.slo, args.period_days)\n\n if args.mode == 'availability':\n if not args.total_requests or args.failed_requests is None:\n print(\"❌ --total-requests and --failed-requests required\")\n sys.exit(1)\n\n results = calculator.calculate_availability_slo(args.total_requests, args.failed_requests)\n print_availability_results(results)\n\n elif args.mode == 'latency':\n if not args.total_requests or args.slow_requests is None:\n print(\"❌ --total-requests and --slow-requests required\")\n sys.exit(1)\n\n results = calculator.calculate_latency_slo(args.total_requests, args.slow_requests)\n print_availability_results(results) # Same format\n\n elif args.mode == 'burn-rate':\n if not all([args.errors is not None, args.requests, args.window_hours]):\n print(\"❌ --errors, --requests, and --window-hours required\")\n sys.exit(1)\n\n results = calculator.calculate_burn_rate(args.errors, args.requests, args.window_hours)\n print_burn_rate_results(results)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":13515,"content_sha256":"c26ab0f0a31e5efa830a9f24938ec356bfaef927438bd47b95f4ad0015cff662"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Monitoring & Observability","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Core Workflow: Observability Implementation","type":"text"}]},{"type":"paragraph","content":[{"text":"Use this decision tree to determine your starting point:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":""},"content":[{"text":"Are you setting up monitoring from scratch?\n├─ YES → Start with \"1. Design Metrics Strategy\"\n└─ NO → Do you have an existing issue?\n ├─ YES → Go to \"9. Troubleshooting & Analysis\"\n └─ NO → Are you improving existing monitoring?\n ├─ Alerts → Go to \"3. Alert Design\"\n ├─ Dashboards → Go to \"4. Dashboard & Visualization\"\n ├─ SLOs → Go to \"5. SLO & Error Budgets\"\n ├─ Tool selection → Read references/tool_comparison.md\n └─ Using Datadog? High costs? → Go to \"7. Datadog Cost Optimization & Migration\"","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"1. Design Metrics Strategy","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Start with The Four Golden Signals","type":"text"}]},{"type":"paragraph","content":[{"text":"Every service should monitor: ","type":"text"},{"text":"Latency","type":"text","marks":[{"type":"strong"}]},{"text":" (p50/p95/p99), ","type":"text"},{"text":"Traffic","type":"text","marks":[{"type":"strong"}]},{"text":" (req/s), ","type":"text"},{"text":"Errors","type":"text","marks":[{"type":"strong"}]},{"text":" (failure rate), ","type":"text"},{"text":"Saturation","type":"text","marks":[{"type":"strong"}]},{"text":" (resource utilization).","type":"text"}]},{"type":"paragraph","content":[{"text":"RED Method","type":"text","marks":[{"type":"strong"}]},{"text":" (request-driven): Rate, Errors, Duration | ","type":"text"},{"text":"USE Method","type":"text","marks":[{"type":"strong"}]},{"text":" (infrastructure): Utilization, Saturation, Errors","type":"text"}]},{"type":"paragraph","content":[{"text":"Quick Start - Web Application Example","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"promql"},"content":[{"text":"# Rate (requests/sec)\nsum(rate(http_requests_total[5m]))\n\n# Errors (error rate %)\nsum(rate(http_requests_total{status=~\"5..\"}[5m]))\n /\nsum(rate(http_requests_total[5m])) * 100\n\n# Duration (p95 latency)\nhistogram_quantile(0.95,\n sum(rate(http_request_duration_seconds_bucket[5m])) by (le)\n)","type":"text"}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/metrics_design.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — metric types, cardinality, naming conventions, dashboard design","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Querying Metrics Directly","type":"text"}]},{"type":"paragraph","content":[{"text":"Query Prometheus and CloudWatch metrics using CLI/curl:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Query Prometheus instant value\ncurl -s 'http://localhost:9090/api/v1/query?query=rate(http_requests_total[5m])' | jq .\n\n# Query Prometheus over a time range (last 1h, 15s step)\ncurl -s 'http://localhost:9090/api/v1/query_range?query=rate(http_requests_total[5m])&start='$(date -u -v-1H +%Y-%m-%dT%H:%M:%SZ)'&end='$(date -u +%Y-%m-%dT%H:%M:%SZ)'&step=15s' | jq .\n\n# Query CloudWatch metrics (e.g., EC2 CPU over last 1 hour)\naws cloudwatch get-metric-statistics \\\n --namespace AWS/EC2 \\\n --metric-name CPUUtilization \\\n --dimensions Name=InstanceId,Value=i-1234567890abcdef0 \\\n --start-time $(date -u -v-1H +%Y-%m-%dT%H:%M:%SZ) \\\n --end-time $(date -u +%Y-%m-%dT%H:%M:%SZ) \\\n --period 300 \\\n --statistics Average Maximum","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"2. Log Aggregation & Analysis","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Structured Logging Checklist","type":"text"}]},{"type":"paragraph","content":[{"text":"Every log entry should include: timestamp (ISO 8601), log level, message, service name, request ID (for tracing).","type":"text"}]},{"type":"paragraph","content":[{"text":"Example structured log (JSON)","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"json"},"content":[{"text":"{\n \"timestamp\": \"2024-10-28T14:32:15Z\",\n \"level\": \"error\",\n \"message\": \"Payment processing failed\",\n \"service\": \"payment-service\",\n \"request_id\": \"550e8400-e29b-41d4-a716-446655440000\",\n \"user_id\": \"user123\",\n \"order_id\": \"ORD-456\",\n \"error_type\": \"GatewayTimeout\",\n \"duration_ms\": 5000\n}","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Log Analysis","type":"text"}]},{"type":"paragraph","content":[{"text":"Analyze logs for errors, patterns, and anomalies:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Analyze log file for patterns\npython3 scripts/log_analyzer.py application.log\n\n# Show error lines with context\npython3 scripts/log_analyzer.py application.log --show-errors\n\n# Extract stack traces\npython3 scripts/log_analyzer.py application.log --show-traces","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Script","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"scripts/log_analyzer.py","type":"text","marks":[{"type":"link","attrs":{"href":"scripts/log_analyzer.py","title":null}}]}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/logging_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — structured logging, aggregation patterns (ELK, Loki, CloudWatch), PII redaction, sampling","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"3. Alert Design","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Alert Design Principles","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Actionable","type":"text","marks":[{"type":"strong"}]},{"text":" - If you can't act on it, don't alert","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Symptom-based","type":"text","marks":[{"type":"strong"}]},{"text":" - Alert on user experience, not components","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"SLO-tied","type":"text","marks":[{"type":"strong"}]},{"text":" - Connect to business impact","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Low noise","type":"text","marks":[{"type":"strong"}]},{"text":" - Only page for critical issues","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Alert Severity Levels","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Severity","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Response Time","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Example","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Critical","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Page immediately","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Service down, SLO violation","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Warning","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Ticket, review in hours","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Elevated error rate, resource warning","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Info","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Log for awareness","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Deployment completed, scaling event","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Multi-Window Burn Rate Alerting","type":"text"}]},{"type":"paragraph","content":[{"text":"Alert when error budget is consumed too quickly:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"yaml"},"content":[{"text":"# Fast burn (1h window) - Critical\n- alert: ErrorBudgetFastBurn\n expr: |\n (error_rate / 0.001) > 14.4 # 99.9% SLO\n for: 2m\n labels:\n severity: critical\n\n# Slow burn (6h window) - Warning\n- alert: ErrorBudgetSlowBurn\n expr: |\n (error_rate / 0.001) > 6 # 99.9% SLO\n for: 30m\n labels:\n severity: warning","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Alert Quality Checker","type":"text"}]},{"type":"paragraph","content":[{"text":"Audit your alert rules against best practices:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Check single file\npython3 scripts/alert_quality_checker.py alerts.yml\n\n# Check all rules in directory\npython3 scripts/alert_quality_checker.py /path/to/prometheus/rules/","type":"text"}]},{"type":"paragraph","content":[{"text":"Checks","type":"text","marks":[{"type":"strong"}]},{"text":": naming conventions, required labels/annotations, PromQL quality, 'for' clause to prevent flapping.","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Script","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"scripts/alert_quality_checker.py","type":"text","marks":[{"type":"link","attrs":{"href":"scripts/alert_quality_checker.py","title":null}}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Alert Templates","type":"text"}]},{"type":"paragraph","content":[{"text":"Production-ready alert rule templates:","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Templates","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"assets/templates/prometheus-alerts/webapp-alerts.yml","type":"text","marks":[{"type":"link","attrs":{"href":"assets/templates/prometheus-alerts/webapp-alerts.yml","title":null}}]},{"text":" - Web application alerts","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"assets/templates/prometheus-alerts/kubernetes-alerts.yml","type":"text","marks":[{"type":"link","attrs":{"href":"assets/templates/prometheus-alerts/kubernetes-alerts.yml","title":null}}]},{"text":" - Kubernetes alerts","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/alerting_best_practices.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — alert design patterns, routing, inhibition rules, runbook structure, on-call practices","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Runbook Template","type":"text"}]},{"type":"paragraph","content":[{"text":"Create comprehensive runbooks for your alerts:","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Template","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"assets/templates/runbooks/incident-runbook-template.md","type":"text","marks":[{"type":"link","attrs":{"href":"assets/templates/runbooks/incident-runbook-template.md","title":null}}]}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"4. Dashboard & Visualization","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Dashboard Design Principles","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Top-down layout","type":"text","marks":[{"type":"strong"}]},{"text":": Most important metrics first","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Color coding","type":"text","marks":[{"type":"strong"}]},{"text":": Red (critical), yellow (warning), green (healthy)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Consistent time windows","type":"text","marks":[{"type":"strong"}]},{"text":": All panels use same time range","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Limit panels","type":"text","marks":[{"type":"strong"}]},{"text":": 8-12 panels per dashboard maximum","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Include context","type":"text","marks":[{"type":"strong"}]},{"text":": Show related metrics together","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Recommended layout","type":"text","marks":[{"type":"strong"}]},{"text":": Overall Health (single stats) -> Request Rate & Errors -> Latency Distribution -> Resource Usage","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Generate Grafana Dashboards","type":"text"}]},{"type":"paragraph","content":[{"text":"Automatically generate dashboards from templates:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Web application dashboard\npython3 scripts/dashboard_generator.py webapp \\\n --title \"My API Dashboard\" \\\n --service my_api \\\n --output dashboard.json\n\n# Kubernetes dashboard\npython3 scripts/dashboard_generator.py kubernetes \\\n --title \"K8s Production\" \\\n --namespace production \\\n --output k8s-dashboard.json\n\n# Database dashboard\npython3 scripts/dashboard_generator.py database \\\n --title \"PostgreSQL\" \\\n --db-type postgres \\\n --instance db.example.com:5432 \\\n --output db-dashboard.json","type":"text"}]},{"type":"paragraph","content":[{"text":"Supports","type":"text","marks":[{"type":"strong"}]},{"text":": Web applications, Kubernetes, Databases (PostgreSQL, MySQL).","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Script","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"scripts/dashboard_generator.py","type":"text","marks":[{"type":"link","attrs":{"href":"scripts/dashboard_generator.py","title":null}}]}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"5. SLO & Error Budgets","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"SLO Fundamentals","type":"text"}]},{"type":"paragraph","content":[{"text":"SLI (measurement), SLO (target), Error Budget (allowed failure = 100% - SLO). See ","type":"text"},{"text":"references/slo_sla_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" for full definitions.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Common SLO Targets","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Availability","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Downtime/Month","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Use Case","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"99%","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"7.2 hours","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Internal tools","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"99.9%","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"43.2 minutes","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Standard production","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"99.95%","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"21.6 minutes","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Critical services","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"99.99%","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"4.3 minutes","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"High availability","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"SLO Calculator","type":"text"}]},{"type":"paragraph","content":[{"text":"Calculate compliance, error budgets, and burn rates:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Show SLO reference table\npython3 scripts/slo_calculator.py --table\n\n# Calculate availability SLO\npython3 scripts/slo_calculator.py availability \\\n --slo 99.9 \\\n --total-requests 1000000 \\\n --failed-requests 1500 \\\n --period-days 30\n\n# Calculate burn rate\npython3 scripts/slo_calculator.py burn-rate \\\n --slo 99.9 \\\n --errors 50 \\\n --requests 10000 \\\n --window-hours 1","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Script","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"scripts/slo_calculator.py","type":"text","marks":[{"type":"link","attrs":{"href":"scripts/slo_calculator.py","title":null}}]}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/slo_sla_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — choosing SLIs, setting targets, error budget policies, burn rate alerting, SLA contracts","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"6. Distributed Tracing","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"When to Use Tracing","type":"text"}]},{"type":"paragraph","content":[{"text":"Use distributed tracing when you need to:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Debug performance issues across services","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Understand request flow through microservices","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Identify bottlenecks in distributed systems","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Find N+1 query problems","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"OpenTelemetry Implementation","type":"text"}]},{"type":"paragraph","content":[{"text":"Python example","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from opentelemetry import trace\n\ntracer = trace.get_tracer(__name__)\n\[email protected]_as_current_span(\"process_order\")\ndef process_order(order_id):\n span = trace.get_current_span()\n span.set_attribute(\"order.id\", order_id)\n\n try:\n result = payment_service.charge(order_id)\n span.set_attribute(\"payment.status\", \"success\")\n return result\n except Exception as e:\n span.set_status(trace.Status(trace.StatusCode.ERROR))\n span.record_exception(e)\n raise","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Sampling Strategies","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Development","type":"text","marks":[{"type":"strong"}]},{"text":": 100% (ALWAYS_ON)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Staging","type":"text","marks":[{"type":"strong"}]},{"text":": 50-100%","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Production","type":"text","marks":[{"type":"strong"}]},{"text":": 1-10% (or error-based sampling)","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Error-based sampling","type":"text","marks":[{"type":"strong"}]},{"text":" (always sample errors, 1% of successes):","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"class ErrorSampler(Sampler):\n def should_sample(self, parent_context, trace_id, name, **kwargs):\n attributes = kwargs.get('attributes', {})\n\n if attributes.get('error', False):\n return Decision.RECORD_AND_SAMPLE\n\n if trace_id & 0xFF \u003c 3: # ~1%\n return Decision.RECORD_AND_SAMPLE\n\n return Decision.DROP","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"OTel Collector Configuration","type":"text"}]},{"type":"paragraph","content":[{"text":"→ Template","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"assets/templates/otel-config/collector-config.yaml","type":"text","marks":[{"type":"link","attrs":{"href":"assets/templates/otel-config/collector-config.yaml","title":null}}]},{"text":" — OTLP/Prometheus/host metrics receivers, batching, tail sampling, multiple exporters (Tempo, Jaeger, Loki, Prometheus, CloudWatch, Datadog)","type":"text"}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/tracing_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — OTel instrumentation (Python, Node.js, Go, Java), context propagation, backend comparison, analysis patterns","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"7. Datadog Cost Optimization & Migration","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scenario 1: I'm Using Datadog and Costs Are Too High","type":"text"}]},{"type":"paragraph","content":[{"text":"Check usage directly in the Datadog UI at ","type":"text"},{"text":"Plan & Usage > Usage Summary","type":"text","marks":[{"type":"strong"}]},{"text":", or query the API:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Get Datadog usage summary for the current month (requires DD_API_KEY and DD_APP_KEY)\ncurl -s -X GET \"https://api.datadoghq.com/api/v1/usage/summary?start_month=$(date -u +%Y-%m)\" \\\n -H \"DD-API-KEY: ${DD_API_KEY}\" \\\n -H \"DD-APPLICATION-KEY: ${DD_APP_KEY}\" | jq .\n\n# Get hourly usage for hosts (identify peak usage)\ncurl -s -X GET \"https://api.datadoghq.com/api/v1/usage/hosts?start_hr=$(date -u -v-24H +%Y-%m-%dT%H)\" \\\n -H \"DD-API-KEY: ${DD_API_KEY}\" \\\n -H \"DD-APPLICATION-KEY: ${DD_APP_KEY}\" | jq .","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Common Cost Optimization Strategies","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Custom Metrics","type":"text","marks":[{"type":"strong"}]},{"text":" (20-40% savings): Remove high-cardinality tags, delete unused metrics, aggregate before sending","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Log Management","type":"text","marks":[{"type":"strong"}]},{"text":" (30-50% savings): Sample high-volume services, exclude debug logs in prod, archive to S3 after 7 days","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"APM","type":"text","marks":[{"type":"strong"}]},{"text":" (15-25% savings): Reduce trace sampling (10% to 5%), remove APM from non-critical services","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Infrastructure","type":"text","marks":[{"type":"strong"}]},{"text":" (10-20% savings): Use container-based pricing, remove agents from ephemeral instances, consolidate staging","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scenario 2: Migrating Away from Datadog","type":"text"}]},{"type":"paragraph","content":[{"text":"Migration to open-source stack (Prometheus + Grafana, Loki, Tempo/Jaeger, Alertmanager). Estimated savings: 60-77%.","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Migration Strategy","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Phase 1","type":"text","marks":[{"type":"strong"}]},{"text":" (Month 1-2): Deploy open-source stack in parallel, migrate metrics first, validate accuracy","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Phase 2","type":"text","marks":[{"type":"strong"}]},{"text":" (Month 2-3): Convert dashboards to Grafana, translate alert rules (DQL to PromQL), train team","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Phase 3","type":"text","marks":[{"type":"strong"}]},{"text":" (Month 3-4): Set up Loki for logs, deploy Tempo/Jaeger for traces, update instrumentation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Phase 4","type":"text","marks":[{"type":"strong"}]},{"text":" (Month 4-5): Confirm all functionality migrated, decommission Datadog","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Full migration guide","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/datadog_migration.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"Query translation","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/dql_promql_translation.md","type":"text","marks":[{"type":"code_inline"}]}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"8. Tool Selection & Comparison","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Solution","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Monthly Cost (100 hosts)","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Best For","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Prometheus + Loki + Tempo","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"$1,500","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Kubernetes, budget-conscious, ops-capable teams","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Grafana Cloud","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"$3,000","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Open-source stack, low ops overhead","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Datadog","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"$8,000","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Ease of use, full observability out of the box","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"ELK Stack","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"$4,000","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Heavy log analysis, powerful search","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"CloudWatch","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"$2,000","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Single AWS provider, simple needs","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"Deep dive","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"references/tool_comparison.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" — full comparison of metrics, logging, tracing, and full-stack platforms","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"9. Troubleshooting & Analysis","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Health Check Validation","type":"text"}]},{"type":"paragraph","content":[{"text":"Validate health check endpoints using ","type":"text"},{"text":"curl","type":"text","marks":[{"type":"code_inline"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Check endpoint returns 200 and measure response time\ncurl -sf -o /dev/null -w \"status:%{http_code} time:%{time_total}s\\n\" https://api.example.com/health\n\n# Get full response body (check for JSON 'status' field, version info)\ncurl -sf https://api.example.com/health | jq .\n\n# Check multiple endpoints in sequence\nfor ep in /health /readiness /liveness; do\n printf \"%-20s \" \"$ep\"\n curl -sf -o /dev/null -w \"status:%{http_code} time:%{time_total}s\\n\" \"https://api.example.com${ep}\"\ndone","type":"text"}]},{"type":"paragraph","content":[{"text":"What to verify","type":"text","marks":[{"type":"strong"}]},{"text":": 200 status, response time \u003c 1s, JSON format with ","type":"text"},{"text":"status","type":"text","marks":[{"type":"code_inline"}]},{"text":" field, version/build info, dependency checks, no caching headers (","type":"text"},{"text":"Cache-Control: no-cache","type":"text","marks":[{"type":"code_inline"}]},{"text":").","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Common Troubleshooting Workflows","type":"text"}]},{"type":"paragraph","content":[{"text":"High Latency","type":"text","marks":[{"type":"strong"}]},{"text":": Check dashboards for spike -> query traces for slow ops -> check DB slow queries -> check external APIs -> review deployments -> check resource utilization","type":"text"}]},{"type":"paragraph","content":[{"text":"High Error Rate","type":"text","marks":[{"type":"strong"}]},{"text":": Check error logs -> identify affected endpoints -> check dependency health -> review deployments -> check resource limits -> verify configuration","type":"text"}]},{"type":"paragraph","content":[{"text":"Service Down","type":"text","marks":[{"type":"strong"}]},{"text":": Check pods/instances running -> check health endpoint -> review deployments -> check resource availability -> check network -> review startup logs","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"Quick Reference Commands","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Prometheus Queries","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"promql"},"content":[{"text":"# Request rate\nsum(rate(http_requests_total[5m]))\n\n# Error rate\nsum(rate(http_requests_total{status=~\"5..\"}[5m]))\n /\nsum(rate(http_requests_total[5m])) * 100\n\n# P95 latency\nhistogram_quantile(0.95,\n sum(rate(http_request_duration_seconds_bucket[5m])) by (le)\n)\n\n# CPU usage\n100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)\n\n# Memory usage\n(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100","type":"text"}]},{"type":"paragraph","content":[{"text":"See ","type":"text"},{"text":"references/quick_commands.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" for Kubernetes, Elasticsearch, Loki, and CloudWatch query references.","type":"text"}]},{"type":"hr","attrs":{"markup":"---"}},{"type":"heading","attrs":{"level":2},"content":[{"text":"Resources Summary","type":"text"}]},{"type":"paragraph","content":[{"text":"Scripts","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"alert_quality_checker.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"dashboard_generator.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"log_analyzer.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"slo_calculator.py","type":"text","marks":[{"type":"code_inline"}]}]},{"type":"paragraph","content":[{"text":"References","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"metrics_design.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"alerting_best_practices.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"logging_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"tracing_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"slo_sla_guide.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"tool_comparison.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"datadog_migration.md","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"dql_promql_translation.md","type":"text","marks":[{"type":"code_inline"}]}]},{"type":"paragraph","content":[{"text":"Templates","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"prometheus-alerts/webapp-alerts.yml","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"prometheus-alerts/kubernetes-alerts.yml","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"otel-config/collector-config.yaml","type":"text","marks":[{"type":"code_inline"}]},{"text":" | ","type":"text"},{"text":"runbooks/incident-runbook-template.md","type":"text","marks":[{"type":"code_inline"}]}]}]},"metadata":{"date":"2026-11-02","name":"monitoring-observability","author":"@skillopedia","source":{"stars":206,"repo_name":"devops-claude-skills","origin_url":"https://github.com/ahmedasmar/devops-claude-skills/blob/HEAD/monitoring-observability/skills/SKILL.md","repo_owner":"ahmedasmar","body_sha256":"7a1dea9f93ac92fbf28f94bfa149c95ce7d387ea1b657383d2b55a08a2b9da63","cluster_key":"1a96566101a9a1d2c4eaf17c20ce9ab6cf4b1460b0f83b6f375ac3783014c226","clean_bundle":{"format":"clean-skill-bundle-v1","source":"ahmedasmar/devops-claude-skills/monitoring-observability/skills/SKILL.md","text_bytes":191842,"source_commit":"1489c33ad8829a11219e423327d6b59f8339cee4","skills_sh_rank":19971,"attachment_count":17,"text_attachments":17,"attachment_storage":"skill_attachments","binary_attachments":0},"cluster_size":1,"skill_md_path":"monitoring-observability/skills/SKILL.md","import_metadata":{"date":"2026-11-02","author":"@skillopedia","version":"v1","category":"devops-infrastructure","category_label":"DevOps"},"exact_dupes_collapsed_into_this":0},"version":"v1","category":"devops-infrastructure","import_tag":"skills-sh-top20k-2026-11","description":"Monitoring and observability strategy, implementation, and troubleshooting. Use this skill whenever the user mentions monitoring, observability, metrics, logs, traces, alerting, SLOs, Prometheus, Grafana, Datadog, Loki, or OpenTelemetry. Triggers include designing metrics strategy (Four Golden Signals, RED/USE), setting up Prometheus/Grafana/Loki, creating alerts or dashboards, calculating SLOs and error budgets, instrumenting with OpenTelemetry, analyzing performance issues, choosing between monitoring tools, optimizing Datadog costs, migrating to open-source stack, and setting up distributed tracing."}},"renderedAt":1793671779963}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.