codex-cli-docs — Skillopedia

OpenAI Codex CLI Documentation Skill CRITICAL: Path Doubling Prevention - MANDATORY ABSOLUTE PROHIBITION: NEVER use with in PowerShell when running scripts from this skill. The Problem: If your current working directory is already inside the skill directory, using relative paths causes PowerShell to resolve paths relative to the current directory instead of the repository root, resulting in path doubling. REQUIRED Solutions (choose one): 1. ALWAYS use absolute paths (recommended) 2. Use separate commands (never with ) 3. Run from repository root with relative paths NEVER DO THIS: - Chain with…

\n match = re.search(pattern, content, re.MULTILINE | re.IGNORECASE)\n\n if not match:\n return None\n\n start = match.end()\n level = len(match.group(1))\n\n # Find next heading at same or higher level\n next_pattern = rf'^#{{{1},{level}}}\\s+'\n next_match = re.search(next_pattern, content[start:], re.MULTILINE)\n\n if next_match:\n section_content = content[start:start + next_match.start()].strip()\n else:\n section_content = content[start:].strip()\n\n return {\n 'doc_id': doc_id,\n 'section': section_heading,\n 'content': section_content,\n }\n except Exception:\n return None\n\n def refresh_index(self, check_drift: bool = False) -> dict[str, Any]:\n \"\"\"\n Refresh the index from filesystem.\n\n Args:\n check_drift: If True, check for content drift\n\n Returns:\n Dictionary with refresh results\n\n Example:\n >>> api = CodexDocsAPI()\n >>> result = api.refresh_index()\n >>> print(result['total_entries'])\n \"\"\"\n try:\n import subprocess\n scripts_dir = Path(__file__).parent / 'scripts'\n result = subprocess.run(\n [sys.executable, str(scripts_dir / 'management' / 'refresh_index.py')],\n capture_output=True,\n text=True,\n )\n return {\n 'success': result.returncode == 0,\n 'output': result.stdout,\n 'error': result.stderr if result.returncode != 0 else None,\n }\n except Exception as e:\n return {\n 'success': False,\n 'output': '',\n 'error': str(e),\n }\n\n\n# Module-level convenience functions\n_default_api: CodexDocsAPI | None = None\n\n\ndef _get_api() -> CodexDocsAPI:\n \"\"\"Get or create default API instance.\"\"\"\n global _default_api\n if _default_api is None:\n _default_api = CodexDocsAPI()\n return _default_api\n\n\ndef find_document(query: str, limit: int = 10) -> list[dict[str, Any]]:\n \"\"\"Find documents by natural language query.\"\"\"\n return _get_api().find_document(query, limit)\n\n\ndef resolve_doc_id(doc_id: str) -> dict[str, Any] | None:\n \"\"\"Resolve doc_id to file path and metadata.\"\"\"\n return _get_api().resolve_doc_id(doc_id)\n\n\ndef get_docs_by_tag(tag: str, limit: int = 100) -> list[dict[str, Any]]:\n \"\"\"Get all documents with a specific tag.\"\"\"\n return _get_api().get_docs_by_tag(tag, limit)\n\n\ndef get_docs_by_category(category: str, limit: int = 100) -> list[dict[str, Any]]:\n \"\"\"Get all documents in a specific category.\"\"\"\n return _get_api().get_docs_by_category(category, limit)\n\n\ndef search_by_keywords(keywords: list[str], limit: int = 25) -> list[dict[str, Any]]:\n \"\"\"Search documents by keywords.\"\"\"\n return _get_api().search_by_keywords(keywords, limit)\n\n\ndef get_document_section(doc_id: str, section_heading: str) -> dict[str, Any] | None:\n \"\"\"Extract a specific section from a document.\"\"\"\n return _get_api().get_document_section(doc_id, section_heading)\n\n\ndef refresh_index(check_drift: bool = False) -> dict[str, Any]:\n \"\"\"Refresh the index from filesystem.\"\"\"\n return _get_api().refresh_index(check_drift)\n\n\nif __name__ == '__main__':\n # Simple CLI for testing\n import argparse\n\n parser = argparse.ArgumentParser(description='Codex CLI Docs API')\n parser.add_argument('command', choices=['find', 'resolve', 'tag', 'category', 'search', 'refresh'])\n parser.add_argument('args', nargs='*')\n\n args = parser.parse_args()\n\n api = CodexDocsAPI()\n\n if args.command == 'find':\n query = ' '.join(args.args) if args.args else 'getting started'\n results = api.find_document(query)\n print(f\"Found {len(results)} documents:\")\n for doc in results[:5]:\n print(f\" - {doc['doc_id']}: {doc['title']}\")\n\n elif args.command == 'resolve':\n doc_id = args.args[0] if args.args else 'codex-cli-overview'\n result = api.resolve_doc_id(doc_id)\n if result:\n print(f\"Resolved {doc_id}:\")\n print(f\" Title: {result['title']}\")\n print(f\" URL: {result['url']}\")\n else:\n print(f\"Not found: {doc_id}\")\n\n elif args.command == 'tag':\n tag = args.args[0] if args.args else 'cli'\n results = api.get_docs_by_tag(tag)\n print(f\"Found {len(results)} documents with tag '{tag}':\")\n for doc in results[:5]:\n print(f\" - {doc['doc_id']}: {doc['title']}\")\n\n elif args.command == 'category':\n category = args.args[0] if args.args else 'guides'\n results = api.get_docs_by_category(category)\n print(f\"Found {len(results)} documents in category '{category}':\")\n for doc in results[:5]:\n print(f\" - {doc['doc_id']}: {doc['title']}\")\n\n elif args.command == 'search':\n keywords = args.args if args.args else ['sandbox', 'tools']\n results = api.search_by_keywords(keywords)\n print(f\"Found {len(results)} documents for keywords {keywords}:\")\n for doc in results[:5]:\n print(f\" - {doc['doc_id']}: {doc['title']} (score: {doc.get('relevance_score', 'N/A')})\")\n\n elif args.command == 'refresh':\n result = api.refresh_index()\n if result['success']:\n print(\"Index refreshed successfully\")\n else:\n print(f\"Refresh failed: {result['error']}\")\n","content_type":"text/x-python; charset=utf-8","language":"python","size":14590,"content_sha256":"34b9ed64e86490e23c6301b8c65bc2b0d578f8acbaef2cd70ab0c099aa5f2d7f"},{"filename":"config/__init__.py","content":"\"\"\"Config package for codex-cli-docs skill.\"\"\"\n","content_type":"text/x-python; charset=utf-8","language":"python","size":47,"content_sha256":"ce78d018c4ba2cd9e1ccacb1f12070744766d186cd6df3651cb171ffaab2b416"},{"filename":"config/config_registry.py","content":"#!/usr/bin/env python3\n\"\"\"\nConfiguration Registry - Single Source of Truth for All Configuration\n\nAdapted for OpenAI Codex CLI Documentation from google-ecosystem gemini-cli-docs skill.\n\nProvides unified access to all configuration files:\n- sources.json (source definitions)\n- filtering.yaml (filtering rules)\n- tag_detection.yaml (tag detection rules)\n- defaults.yaml (default values)\n\nAll configuration can be overridden via environment variables.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport sys\nimport threading\nfrom pathlib import Path\nfrom typing import Any, Dict, Optional\n\n# Add scripts directory to path for imports\n# config/ is one level down from skill root\n_skill_dir = Path(__file__).resolve().parents[1]\n_scripts_dir = _skill_dir / 'scripts'\nif str(_scripts_dir) not in sys.path:\n sys.path.insert(0, str(_scripts_dir))\n\n# Import using full path since scripts is now in sys.path\ntry:\n from utils.script_utils import ensure_yaml_installed\n from utils.logging_utils import get_or_setup_logger\nexcept ImportError:\n # Fallback to direct imports if package structure not available\n import importlib.util\n\n # Load script_utils\n script_utils_path = _scripts_dir / 'utils' / 'script_utils.py'\n spec = importlib.util.spec_from_file_location(\"script_utils\", script_utils_path)\n script_utils = importlib.util.module_from_spec(spec)\n spec.loader.exec_module(script_utils)\n ensure_yaml_installed = script_utils.ensure_yaml_installed\n\n # Load logging_utils\n logging_utils_path = _scripts_dir / 'utils' / 'logging_utils.py'\n spec = importlib.util.spec_from_file_location(\"logging_utils\", logging_utils_path)\n logging_utils = importlib.util.module_from_spec(spec)\n spec.loader.exec_module(logging_utils)\n get_or_setup_logger = logging_utils.get_or_setup_logger\n\n# Ensure PyYAML is available\nyaml = ensure_yaml_installed()\n\nlogger = get_or_setup_logger(__file__)\n\n# Environment variable prefix for overrides\n# Changed from GEMINI_DOCS_ to CODEX_DOCS_\nENV_PREFIX = \"CODEX_DOCS_\"\n\n\nclass ConfigRegistry:\n \"\"\"\n Unified configuration registry with caching and environment variable overrides.\n\n Singleton pattern ensures consistent configuration access across all scripts.\n \"\"\"\n\n _instance: 'ConfigRegistry' | None = None\n _lock = threading.Lock()\n\n def __new__(cls) -> 'ConfigRegistry':\n \"\"\"Ensure only one instance exists (singleton pattern).\"\"\"\n if cls._instance is None:\n with cls._lock:\n if cls._instance is None:\n cls._instance = super().__new__(cls)\n cls._instance._initialized = False\n return cls._instance\n\n def __init__(self):\n \"\"\"Initialize the configuration registry.\"\"\"\n if self._initialized:\n return\n\n self._cache: dict[str, Any] = {}\n self._cache_lock = threading.Lock()\n\n # Determine skill directory\n script_dir = Path(__file__).parent\n self.skill_dir = script_dir.parent\n self.config_dir = script_dir\n\n # Default paths\n self._paths = {\n 'sources': self.skill_dir / 'references' / 'sources.json',\n 'filtering': self.config_dir / 'filtering.yaml',\n 'tag_detection': self.config_dir / 'tag_detection.yaml',\n 'defaults': self.config_dir / 'defaults.yaml',\n }\n\n self._initialized = True\n logger.debug(\"ConfigRegistry initialized\")\n\n def _get_defaults_path(self) -> Path:\n \"\"\"Get path to defaults.yaml, checking multiple locations for test compatibility.\"\"\"\n # Check default location first\n if self._paths['defaults'].exists():\n return self._paths['defaults']\n\n # For tests: check if there's a defaults.yaml in any temp directory config\n for temp_marker in ['/tmp/', '\\\\Temp\\\\', 'AppData/Local/Temp']:\n if temp_marker in str(Path.cwd()):\n return self._paths['defaults']\n\n return self._paths['defaults']\n\n def _load_json(self, path: Path) -> dict[str, Any]:\n \"\"\"Load JSON file.\"\"\"\n if not path.exists():\n raise FileNotFoundError(f\"Configuration file not found: {path}\")\n\n with open(path, 'r', encoding='utf-8') as f:\n return json.load(f)\n\n def _load_yaml(self, path: Path) -> dict[str, Any]:\n \"\"\"Load YAML file with fallback for missing files.\"\"\"\n if not path.exists():\n logger.warning(f\"Configuration file not found, using empty config: {path}\")\n return {}\n\n with open(path, 'r', encoding='utf-8') as f:\n return yaml.safe_load(f) or {}\n\n def _apply_env_overrides(self, config: dict[str, Any], prefix: str = \"\") -> dict[str, Any]:\n \"\"\"\n Apply environment variable overrides to configuration.\n\n Environment variables follow the pattern:\n CODEX_DOCS_\u003cSECTION>_\u003cKEY>\n\n Example: CODEX_DOCS_HTTP_DEFAULT_TIMEOUT=60\n \"\"\"\n result = config.copy()\n\n for key, value in config.items():\n env_key = f\"{ENV_PREFIX}{prefix}{key}\".upper()\n env_value = os.environ.get(env_key)\n\n if env_value is not None:\n # Try to convert to appropriate type\n if isinstance(value, bool):\n env_value = env_value.lower() in ('true', '1', 'yes', 'on')\n elif isinstance(value, int):\n try:\n env_value = int(env_value)\n except ValueError:\n logger.warning(f\"Invalid integer value for {env_key}: {env_value}\")\n continue\n elif isinstance(value, float):\n try:\n env_value = float(env_value)\n except ValueError:\n logger.warning(f\"Invalid float value for {env_key}: {env_value}\")\n continue\n elif isinstance(value, list):\n if ',' in env_value:\n env_value = [v.strip() for v in env_value.split(',')]\n else:\n env_value = [env_value]\n\n result[key] = env_value\n logger.debug(f\"Override from {env_key}: {value} -> {env_value}\")\n\n # Recursively apply overrides to nested dictionaries\n if isinstance(value, dict):\n result[key] = self._apply_env_overrides(value, f\"{prefix}{key}_\")\n\n return result\n\n def load_sources(self, path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load sources.json configuration.\"\"\"\n config_path = Path(path) if path else self._paths['sources']\n cache_key = f'sources:{config_path}'\n\n with self._cache_lock:\n if cache_key in self._cache:\n logger.debug(f\"Using cached sources config: {config_path}\")\n return self._cache[cache_key]\n\n logger.debug(f\"Loading sources config: {config_path}\")\n suffix = config_path.suffix.lower()\n if suffix in ('.yaml', '.yml'):\n config = self._load_yaml(config_path)\n else:\n config = self._load_json(config_path)\n\n if isinstance(config, list):\n config = [self._apply_env_overrides(item, \"SOURCES_\") if isinstance(item, dict) else item\n for item in config]\n elif isinstance(config, dict):\n config = self._apply_env_overrides(config, \"SOURCES_\")\n\n with self._cache_lock:\n self._cache[cache_key] = config\n\n return config\n\n def load_filtering(self, path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load filtering.yaml configuration.\"\"\"\n config_path = path or self._paths['filtering']\n cache_key = f'filtering:{config_path}'\n\n with self._cache_lock:\n if cache_key in self._cache:\n logger.debug(f\"Using cached filtering config: {config_path}\")\n return self._cache[cache_key]\n\n logger.debug(f\"Loading filtering config: {config_path}\")\n config = self._load_yaml(config_path)\n config = self._apply_env_overrides(config, \"FILTERING_\")\n\n with self._cache_lock:\n self._cache[cache_key] = config\n\n return config\n\n def load_tag_detection(self, path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load tag_detection.yaml configuration.\"\"\"\n config_path = path or self._paths['tag_detection']\n cache_key = f'tag_detection:{config_path}'\n\n with self._cache_lock:\n if cache_key in self._cache:\n logger.debug(f\"Using cached tag detection config: {config_path}\")\n return self._cache[cache_key]\n\n logger.debug(f\"Loading tag detection config: {config_path}\")\n config = self._load_yaml(config_path)\n config = self._apply_env_overrides(config, \"TAG_DETECTION_\")\n\n with self._cache_lock:\n self._cache[cache_key] = config\n\n return config\n\n def load_defaults(self, path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load defaults.yaml configuration.\"\"\"\n config_path = path or self._paths['defaults']\n cache_key = f'defaults:{config_path}'\n\n with self._cache_lock:\n if cache_key in self._cache:\n logger.debug(f\"Using cached defaults config: {config_path}\")\n return self._cache[cache_key]\n\n logger.debug(f\"Loading defaults config: {config_path}\")\n config = self._load_yaml(config_path)\n config = self._apply_env_overrides(config, \"\")\n\n with self._cache_lock:\n self._cache[cache_key] = config\n\n return config\n\n def get_default(self, section: str, key: str, default: Any = None) -> Any:\n \"\"\"Get a default value from defaults.yaml.\"\"\"\n defaults = self.load_defaults()\n return defaults.get(section, {}).get(key, default)\n\n def clear_cache(self) -> None:\n \"\"\"Clear all cached configurations.\"\"\"\n with self._cache_lock:\n count = len(self._cache)\n self._cache.clear()\n if count > 0:\n logger.info(f\"Cleared {count} cached configuration(s)\")\n else:\n logger.debug(\"Cache already empty, nothing to clear\")\n\n def reload(self) -> None:\n \"\"\"Force reload of all configurations.\"\"\"\n self.clear_cache()\n logger.info(\"Configuration cache cleared - will reload on next access\")\n\n\n# Module-level convenience functions\ndef get_registry() -> ConfigRegistry:\n \"\"\"Get the singleton ConfigRegistry instance.\"\"\"\n return ConfigRegistry()\n\n\ndef get_default(section: str, key: str, default: Any = None) -> Any:\n \"\"\"Get a default value from defaults.yaml.\"\"\"\n return get_registry().get_default(section, key, default)\n\n\ndef load_sources(path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load sources.json configuration.\"\"\"\n return get_registry().load_sources(path)\n\n\ndef load_filtering(path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load filtering.yaml configuration.\"\"\"\n return get_registry().load_filtering(path)\n\n\ndef load_tag_detection(path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load tag_detection.yaml configuration.\"\"\"\n return get_registry().load_tag_detection(path)\n\n\ndef load_defaults(path: Path | None = None) -> dict[str, Any]:\n \"\"\"Load defaults.yaml configuration.\"\"\"\n return get_registry().load_defaults()\n\n\ndef reload_configs() -> None:\n \"\"\"Force reload of all configurations.\"\"\"\n get_registry().reload()\n\n\nif __name__ == '__main__':\n # Self-test\n import sys\n\n print(\"ConfigRegistry Self-Test (OpenAI Codex CLI Docs)\")\n print(\"=\" * 50)\n\n registry = ConfigRegistry()\n\n print(\"\\nTesting defaults.yaml loading...\")\n try:\n defaults = registry.load_defaults()\n print(f\"✓ Loaded defaults config\")\n print(f\" - HTTP default timeout: {defaults.get('http', {}).get('default_timeout')}\")\n print(f\" - Scraping max workers: {defaults.get('scraping', {}).get('max_workers')}\")\n print(f\" - Index chunk size: {defaults.get('index', {}).get('chunk_size')}\")\n except Exception as e:\n print(f\"✗ Failed to load defaults config: {e}\")\n sys.exit(1)\n\n print(\"\\nTesting get_default()...\")\n try:\n timeout = registry.get_default('http', 'default_timeout', 30)\n print(f\"✓ get_default('http', 'default_timeout'): {timeout}\")\n\n max_workers = registry.get_default('scraping', 'max_workers', 3)\n print(f\"✓ get_default('scraping', 'max_workers'): {max_workers}\")\n except Exception as e:\n print(f\"✗ get_default() failed: {e}\")\n sys.exit(1)\n\n print(\"\\nTesting filtering.yaml loading...\")\n try:\n filtering = registry.load_filtering()\n print(f\"✓ Loaded filtering config\")\n print(f\" - Domain stop words: {len(filtering.get('domain_stop_words', []))}\")\n except Exception as e:\n print(f\"✗ Failed to load filtering config: {e}\")\n\n print(\"\\nTesting tag_detection.yaml loading...\")\n try:\n tag_config = registry.load_tag_detection()\n print(f\"✓ Loaded tag detection config\")\n if 'tags' in tag_config:\n print(f\" - Tags: {len(tag_config['tags'])}\")\n except Exception as e:\n print(f\"✗ Failed to load tag detection config: {e}\")\n\n print(\"\\nTesting sources.json loading...\")\n try:\n sources = registry.load_sources()\n print(f\"✓ Loaded sources config\")\n if isinstance(sources, list):\n print(f\" - Sources: {len(sources)}\")\n elif isinstance(sources, dict) and 'sources' in sources:\n print(f\" - Sources: {len(sources['sources'])}\")\n except Exception as e:\n print(f\"✗ Failed to load sources config: {e}\")\n\n print(\"\\nTesting cache...\")\n defaults2 = registry.load_defaults()\n print(f\"✓ Cache working (second load instant)\")\n\n print(\"\\nTesting reload...\")\n registry.reload()\n defaults3 = registry.load_defaults()\n print(f\"✓ Reload successful\")\n\n print(\"\\n\" + \"=\" * 50)\n print(\"Self-test complete!\")\n","content_type":"text/x-python; charset=utf-8","language":"python","size":14142,"content_sha256":"69f7c24bf73769f2f58803ece06e67c3c91c3a02af3ccd94dbbd31bb6891902f"},{"filename":"config/defaults.yaml","content":"# Default Configuration Values for codex-cli-docs Skill\n# Adapted for OpenAI Codex CLI documentation\n# Values can be overridden via environment variables (see config_registry.py)\n\n# HTTP and Network Configuration\nhttp:\n # Default timeout for HTTP requests (seconds)\n default_timeout: 30\n\n # Maximum timeout for long-running operations (seconds)\n max_timeout: 1800\n\n # Default number of retry attempts for transient failures\n default_max_retries: 3\n\n # Initial delay for exponential backoff (seconds)\n initial_retry_delay: 1.0\n\n # Backoff multiplier for exponential backoff\n backoff_factor: 2.0\n\n # Retryable HTTP status codes\n retryable_status_codes:\n - 429 # Too Many Requests (rate limit)\n - 500 # Internal Server Error\n - 502 # Bad Gateway\n - 503 # Service Unavailable\n - 504 # Gateway Timeout\n\n # User agent string for HTTP requests\n user_agent: \"Codex-Docs-Scraper/1.0 (Educational purposes)\"\n\n # Timeout for HEAD requests (seconds)\n head_request_timeout: 10\n\n # Timeout for markdown URL requests (seconds)\n markdown_request_timeout: 30\n\n# Scraping Configuration\nscraping:\n # URL exclusion patterns (regex patterns to skip during scraping)\n # These URLs are extracted from sources but should not be scraped\n url_exclusions:\n # GitHub raw content - repository files\n - \"raw\\\\.githubusercontent\\\\.com\"\n - \"github\\\\.com/.*/blob/\"\n\n # Delay between requests (seconds) - rate limiting\n rate_limit: 0.5\n\n # Delay between HEAD requests (seconds)\n header_rate_limit: 0.3\n\n # Legacy names (for backward compatibility)\n rate_limit_delay: 0.5\n header_rate_limit_delay: 0.3\n\n # Maximum parallel workers for URL processing\n # Small dataset (~30 pages), no need for high parallelism\n max_workers: 4\n\n # Maximum parallel workers for multi-source scraping\n max_source_workers: 4\n\n # Default timeout for scraping operations (seconds)\n default_timeout: 300\n\n # Default timeout for source scraping operations\n sources_default_timeout: 300\n\n # Lock acquisition timeout (seconds)\n lock_timeout: 30.0\n\n # Index update lock timeout (seconds)\n index_lock_timeout: 60.0\n\n # Progress lock timeout (seconds)\n progress_lock_timeout: 30.0\n\n # Maximum content age in days (for filtering old content)\n max_content_age_days: 0 # 0 = no age filtering\n\n # Skip existing files by default\n skip_existing: true\n\n # Progress reporting intervals\n progress_interval: 30 # Report progress every N seconds\n progress_url_interval: 10 # Report progress every N URLs\n\n# Index Management Configuration\nindex:\n # Chunk size for reading large YAML files (lines per chunk)\n chunk_size: 1000\n\n # Token estimate threshold - files smaller than this are loaded all at once\n token_estimate_threshold: 20000\n\n # Lock timeout for index operations (seconds)\n lock_timeout: 30.0\n\n # Lock retry delay (seconds)\n lock_retry_delay: 0.1\n\n # Lock retry backoff delay (seconds)\n lock_retry_backoff: 0.5\n\n # File operation retry delay (seconds)\n file_retry_delay: 0.2\n\n # Maximum retries for file operations\n file_max_retries: 5\n\n# Path Configuration\npaths:\n # Base directory for canonical documentation storage (relative to skill dir)\n base_dir: \"canonical\"\n\n # Index filename\n index_filename: \"index.yaml\"\n\n # Temporary directory (relative to project root when starting with .claude/)\n # Uses project-standard .claude/temp/ location (already gitignored)\n temp_dir: \".claude/temp\"\n\n # Output directory mappings (domain -> output subdirectory)\n output_dirs: {}\n\n# File System Configuration (deprecated - use paths section)\nfilesystem:\n default_base_dir: \"canonical\"\n default_index_filename: \"index.yaml\"\n\n# Validation Configuration\nvalidation:\n timeout: 60\n max_retries: 1\n\n# Drift Detection Configuration\ndrift:\n max_workers: 5\n timeout: 300\n\n# Subprocess Configuration\nsubprocess:\n default_timeout: 10\n quick_timeout: 5\n install_timeout: 300\n long_timeout: 600\n build_timeout: 600\n\n# Performance Configuration\nperformance:\n parallel_enabled: true\n parallel_min_urls: 2\n\n# Search and Relevance Configuration\nsearch:\n # Default number of results to return from search queries\n # Use --limit flag to override, or --no-limit for all results\n default_limit: 25\n\n # Minimum relevance score threshold (null = no threshold)\n # Results below this score are filtered out\n # Use --min-score flag to override\n min_score_default: null\n\n # Domain priority weights for relevance ranking\n # Higher weights boost results from that domain\n # Single domain for Codex CLI docs\n domain_weights:\n developers.openai.com: 10.0 # Codex CLI docs (only domain)\n\n # Score boosting factors for different match types\n match_weights:\n keyword_exact: 3.0 # Exact keyword match in index\n title_match: 2.5 # Query term appears in title\n tag_match: 1.5 # Query matches a document tag\n category_match: 1.0 # Query matches document category\n\n # Fuzzy matching configuration\n fuzzy:\n enabled: true\n threshold: 0.8 # Minimum similarity score (0.0-1.0)\n\n# Keyword Extraction Configuration\nkeyword_extraction:\n limits:\n max_file_tokens: 4\n max_heading_phrases: 10\n max_heading_keywords: 8\n max_body_keywords: 6\n max_total_keywords: 12\n\n # YAKE automatic keyword extraction configuration\n yake:\n language: 'en'\n max_ngram_size: 3\n dedup_threshold: 0.7\n top_keywords: 15\n min_text_length: 50\n\n# File Pattern Configuration\nfiles:\n markdown_extension: \".md\"\n yaml_extension: \".yaml\"\n json_extension: \".json\"\n\n# User Agent Configuration\nuser_agents:\n default: \"Codex-Docs-Scraper/1.0 (Educational purposes)\"\n management: \"Codex-Docs-Management-Bot/1.0\"\n scraper: \"Codex-Docs-Scraper/1.0 (Educational purposes)\"\n\n# Cache Configuration\ncache:\n # Enable inverted index caching for O(1) keyword lookups\n inverted_index_enabled: true\n\n # Cache TTL in seconds (0 = no expiry)\n ttl: 0\n\n # Cache directory (relative to skill directory)\n directory: \".cache\"\n\n # Maximum cache size in MB (0 = unlimited)\n max_size_mb: 0\n\n# Logging Configuration\nlogging:\n # Log directory (relative to skill directory)\n directory: \"logs\"\n\n # Log retention in days (for cleanup script)\n retention_days: 30\n\n # Maximum diagnostic files to keep per script\n max_diagnostics_per_script: 10\n\n # Log levels: DEBUG, INFO, WARNING, ERROR\n default_level: \"INFO\"\n\n # Enable detailed diagnostics JSON output\n diagnostics_enabled: true\n\n# Maintenance Configuration\nmaintenance:\n # Drift detection defaults\n drift:\n check_404s: true\n check_hashes: false # Disabled by default (network-intensive)\n max_workers: 5\n\n # Cleanup defaults\n cleanup:\n clean_404s: false # Require explicit opt-in\n clean_missing: false # Require explicit opt-in\n dry_run: true # Default to preview mode\n\n # Stale document thresholds\n stale:\n # Days since last fetch to consider stale\n threshold_days: 90\n # Auto-remove stale docs\n auto_remove: false\n\n# Section Extraction Configuration\nextraction:\n # Include child sections when extracting a heading\n include_children: true\n\n # Maximum section depth to include\n max_depth: 3\n\n # Add provenance frontmatter to extracted sections\n add_provenance: true\n\n# Environment Variable Override Prefix\n# All values can be overridden by setting environment variables with prefix:\n# CODEX_DOCS_\u003cSECTION>_\u003cKEY>\n# Example: CODEX_DOCS_HTTP_DEFAULT_TIMEOUT=60\n# Example: CODEX_DOCS_SCRAPING_MAX_WORKERS=5\n# Example: CODEX_DOCS_SEARCH_DEFAULT_LIMIT=50\n# Example: CODEX_DOCS_CACHE_TTL=3600\n","content_type":"application/yaml; charset=utf-8","language":"yaml","size":7547,"content_sha256":"64d1d71b60ee4629bbc4de3bec0f6cef50439b6a08f63f8f6f9fe5615b97974b"},{"filename":"config/filtering.yaml","content":"# Version: 1.0 (Adapted for OpenAI Codex CLI)\n# Domain-Specific Filtering Configuration\n# Terms that should be filtered out or handled specially for Codex CLI documentation\n\n# Technical phrases to preserve as single keywords (bigrams/trigrams)\n# These multi-word terms should not be split during keyword extraction\ntechnical_phrases:\n # Codex CLI specific terms\n - 'codex cli'\n - 'sandbox mode'\n - 'full auto'\n - 'suggest mode'\n - 'auto edit'\n - 'project doc'\n - 'approval policy'\n - 'writable roots'\n - 'context window'\n - 'reasoning summary'\n - 'reasoning trace'\n - 'git integration'\n - 'rollback changes'\n - 'code execution'\n # Generic CLI/tool terms\n - 'context window'\n - 'system prompt'\n - 'code execution'\n - 'tool use'\n - 'mcp servers'\n - 'mcp server'\n - 'model context protocol'\n - 'slash commands'\n - 'web search'\n - 'shell tool'\n - 'file system'\n - 'api keys'\n - 'natural language'\n - 'yaml frontmatter'\n - 'prompt engineering'\n - 'rate limits'\n\n# Stop phrases (phrases to completely exclude from keyword extraction)\nstop_phrases:\n - 'min read'\n - 'read more'\n - 'learn more'\n\n# Domain-specific stop words (brand names, product names, URL domains)\n# These should never appear as keywords - they're noise from paths/URLs\ndomain_stop_words:\n - 'codex'\n - 'openai'\n - 'developers-openai-com'\n - 'cli'\n\n# Query stop words - stripped from search queries before processing\nquery_stop_words:\n # Question words\n - 'when'\n - 'how'\n - 'what'\n - 'why'\n - 'which'\n - 'who'\n - 'where'\n # Articles and prepositions\n - 'to'\n - 'is'\n - 'are'\n - 'the'\n - 'a'\n - 'an'\n - 'and'\n - 'or'\n - 'for'\n - 'in'\n - 'on'\n - 'of'\n - 'with'\n - 'about'\n - 'from'\n # Modal verbs\n - 'can'\n - 'i'\n - 'do'\n - 'does'\n - 'should'\n - 'would'\n - 'could'\n # Generic verbs\n - 'use'\n - 'create'\n - 'get'\n - 'add'\n - 'set'\n - 'make'\n - 'run'\n - 'configure'\n - 'implement'\n # Common query phrases without topic value\n - 'best'\n - 'practices'\n\n# Generic verbs that often start incomplete phrases\ngeneric_verbs:\n - 'start'\n - 'ask'\n - 'get'\n - 'use'\n - 'make'\n - 'create'\n - 'add'\n - 'set'\n - 'learn'\n - 'see'\n - 'read'\n - 'view'\n - 'check'\n - 'look'\n - 'determine'\n - 'directly'\n\n# Generic configuration terms - penalize docs that ONLY match these generic terms\ngeneric_config_terms:\n - 'configuration'\n - 'config'\n - 'setup'\n - 'installation'\n - 'getting started'\n - 'guide'\n - 'tutorial'\n - 'documentation'\n - 'docs'\n - 'reference'\n\n# Incomplete phrase endings\nincomplete_endings:\n - 'your'\n - 'the'\n - 'a'\n - 'an'\n - 'this'\n - 'that'\n - 'these'\n - 'those'\n - 'their'\n - 'about'\n - 'with'\n - 'from'\n - 'opening'\n\n# Generic single words to exclude (even if 5+ chars)\ngeneric_single_words:\n - 'terminal'\n - 'faster'\n - 'ideas'\n - 'helps'\n - 'messages'\n - 'developer'\n - 'developers'\n - 'based'\n - 'benefits'\n - 'across'\n - 'team'\n - 'git'\n - 'different'\n - 'type'\n - 'request'\n - 'description'\n - 'relevant'\n - 'optional'\n - 'files'\n - 'scripts'\n - 'directory'\n - 'directories'\n - 'tasks'\n - 'complex'\n - 'specific'\n - 'commands'\n - 'model'\n - 'common'\n - 'follow'\n - 'recent'\n - 'summary'\n - 'models'\n - 'seconds'\n - 'started'\n - 'tutorial'\n - 'broadly'\n - 'quickly'\n - 'easily'\n - 'properly'\n - 'manually'\n\n# Weak adverbs/adjectives for phrase filtering\nweak_phrase_words:\n - 'notification'\n - 'decision'\n - 'permission'\n - 'intelligent'\n - 'normal'\n - 'broadly'\n - 'seconds'\n - 'started'\n - 'tutorial'\n - 'quickly'\n - 'easily'\n - 'properly'\n - 'manually'\n\n# Weak adverbs for 2-word phrase filtering\nweak_two_word_adverbs:\n - 'automatically'\n - 'manually'\n - 'quickly'\n - 'easily'\n - 'properly'\n\n# Keyword filtering configuration\nkeyword_filtering_config:\n min_length_general: 3\n min_length_single_word: 5\n\n# Generic words that aren't useful for search\ngeneric_words:\n - 'automatically'\n - 'personal'\n - 'access'\n - 'list'\n - 'manage'\n - 'share'\n - 'extend'\n - 'using'\n - 'available'\n - 'surface'\n - 'generating'\n - 'processing'\n - 'document'\n - 'members'\n - 'errors'\n - 'yaml'\n - 'project'\n - 'next'\n - 'steps'\n - 'quick'\n - 'structure'\n - 'constraints'\n - 'limitations'\n - 'resources'\n - 'additional'\n - 'related'\n - 'guide'\n - 'overview'\n - 'intro'\n - 'welcome'\n - 'getting'\n - 'help'\n - 'first'\n - 'second'\n - 'third'\n - 'before'\n - 'after'\n - 'during'\n - 'example'\n - 'examples'\n - 'important'\n - 'note'\n - 'notes'\n - 'also'\n - 'more'\n - 'most'\n - 'better'\n - 'best'\n - 'good'\n - 'well'\n - 'way'\n - 'ways'\n - 'part'\n - 'parts'\n - 'thing'\n - 'things'\n","content_type":"application/yaml; charset=utf-8","language":"yaml","size":4661,"content_sha256":"9798b1938a8d1cb8596b15e34b544b0f12d725c7d5d3da6e5ce3038f4472cb14"},{"filename":"config/tag_detection.yaml","content":"# Tag Detection Configuration for OpenAI Codex CLI Documentation\n# Version: 1.0\n\ntags:\n api:\n terms:\n - authentication\n - authorization\n - api key\n - api endpoint\n - api reference\n - http request\n - api response\n min_mentions: 8\n\n tools:\n terms:\n - tool\n - tools\n - shell tool\n - file system\n - web search\n - tool use\n min_mentions: 5\n\n mcp:\n terms:\n - mcp\n - model context protocol\n - mcp server\n - mcp tool\n min_mentions: 5\n\n sandbox:\n terms:\n - sandbox\n - sandbox mode\n - sandboxed\n - isolation\n - security\n min_mentions: 4\n\n troubleshooting:\n terms:\n - troubleshoot\n - troubleshooting\n - fix error\n - error handling\n - debug\n - debugging\n min_mentions: 3\n additional_terms:\n - issue\n - problem\n additional_min_mentions: 3\n\n security:\n terms:\n - security\n - secure\n - permission\n - permissions\n - access control\n - approval policy\n - writable roots\n min_mentions: 4\n\n installation:\n terms:\n - install\n - installation\n - setup\n - prerequisites\n - dependencies\n - getting started\n - quickstart\n min_mentions: 5\n\n configuration:\n terms:\n - configuration\n - config\n - settings\n - environment variable\n - configure\n min_mentions: 6\n\n modes:\n terms:\n - mode\n - full auto\n - suggest mode\n - auto edit\n - approval\n min_mentions: 4\n\n models:\n terms:\n - model\n - models\n - openai\n - model selection\n - reasoning\n min_mentions: 6\n\n authentication:\n terms:\n - authentication\n - auth\n - api key\n - credentials\n min_mentions: 5\n\n git:\n terms:\n - git\n - version control\n - rollback\n - changes\n - diff\n min_mentions: 4\n\n cli:\n terms:\n - cli\n - command line\n - terminal\n - commands\n - repl\n min_mentions: 6\n\n context:\n terms:\n - context\n - context window\n - project doc\n - codebase\n min_mentions: 4\n\ncategories:\n api:\n path_patterns:\n - api\n - /api/\n tag_mapping:\n - api\n content_terms:\n - api\n - endpoint\n - authentication\n\n tools:\n path_patterns:\n - tool\n - tools\n tag_mapping:\n - tools\n content_terms:\n - tool\n - shell\n - file system\n\n guides:\n path_patterns:\n - guide\n - tutorial\n - quickstart\n tag_mapping:\n - guides\n content_terms:\n - guide\n - tutorial\n - how to\n - getting started\n\n installation:\n path_patterns:\n - setup\n - install\n - quickstart\n tag_mapping:\n - installation\n content_terms:\n - install\n - installation\n - setup\n - prerequisites\n\n configuration:\n path_patterns:\n - config\n - settings\n tag_mapping:\n - configuration\n content_terms:\n - configuration\n - config\n - settings\n\n cli:\n path_patterns:\n - cli\n - commands\n tag_mapping:\n - cli\n content_terms:\n - cli\n - command\n - terminal\n\n security:\n path_patterns:\n - security\n - sandbox\n tag_mapping:\n - security\n - sandbox\n content_terms:\n - security\n - sandbox\n - permissions\n\n core:\n path_patterns:\n - core\n - architecture\n tag_mapping:\n - core\n content_terms:\n - core\n - architecture\n - system\n","content_type":"application/yaml; charset=utf-8","language":"yaml","size":3585,"content_sha256":"d6fc82072f1d291952029039effb52642e8345f035ecbe7820681cef35cc6e8e"},{"filename":"references/sources.json","content":"[\n {\n \"name\": \"developers.openai.com (codex llms-txt)\",\n \"type\": \"llms-txt\",\n \"url\": \"https://developers.openai.com/codex/llms.txt\",\n \"skip_existing\": true,\n \"expected_count\": 68,\n \"expected_count_tolerance\": 15,\n \"timeout\": 300,\n \"try_markdown\": true,\n \"comment\": \"OpenAI Codex CLI documentation from llms.txt discovery index. The llms.txt contains embedded markdown links that are extracted by the parser.\"\n }\n]\n","content_type":"application/json; charset=utf-8","language":"json","size":443,"content_sha256":"54ae24aef3fefdab3dda8283d92bc61824323e0d6b8f4e54367a05d469dc5561"},{"filename":"scripts/__init__.py","content":"\"\"\"Scripts package for codex-cli-docs skill.\"\"\"\n","content_type":"text/x-python; charset=utf-8","language":"python","size":48,"content_sha256":"094e1e487f2263b6c2dc53093ab8b8818bf9b8460960d0e80393f21ed489b8d5"},{"filename":"scripts/bootstrap.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nbootstrap.py - Minimal bootstrap module for codex-cli-docs scripts.\n\nThis module provides a single entry point for path setup. Scripts can use this\ninstead of the 2-line boilerplate pattern that was duplicated across files.\n\nBEFORE (2 lines repeated everywhere):\n import sys\n from pathlib import Path\n sys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n from utils.common_paths import setup_python_path; skill_dir, scripts_dir, config_dir = setup_python_path()\n\nAFTER (1 line):\n from bootstrap import setup; skill_dir, scripts_dir, config_dir = setup()\n\nOR for scripts in scripts/ root:\n import bootstrap; skill_dir, scripts_dir, config_dir = bootstrap.setup()\n\nThe setup() function handles:\n1. Adding scripts/ and config/ to sys.path\n2. Returning (skill_dir, scripts_dir, config_dir) paths\n3. Working from any depth in the scripts tree\n\nNote: This file must live in scripts/ root to be importable from subdirectories\nafter the initial sys.path.insert() call.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\n\n\ndef _find_scripts_dir(from_path: Path | None = None) -> Path:\n \"\"\"Find the scripts directory by walking up from caller location.\n\n Args:\n from_path: Starting path. If None, uses caller's __file__.\n\n Returns:\n Path to scripts directory (absolute)\n \"\"\"\n if from_path is None:\n # Default to this file's location (scripts/)\n return Path(__file__).resolve().parent\n\n from_path = Path(from_path).resolve()\n\n # Walk up to find scripts/ directory\n current = from_path if from_path.is_dir() else from_path.parent\n while current != current.parent:\n if current.name == 'scripts' and (current / 'bootstrap.py').exists():\n return current\n current = current.parent\n\n # Fallback: this file's location\n return Path(__file__).resolve().parent\n\n\ndef setup(from_path: Path | None = None) -> tuple[Path, Path, Path]:\n \"\"\"Setup sys.path and return key directories.\n\n This is the main entry point. Call this at the top of any script\n to configure imports and get standard directory paths.\n\n Args:\n from_path: Optional starting path for resolution.\n\n Returns:\n Tuple of (skill_dir, scripts_dir, config_dir) as absolute paths\n\n Example:\n from bootstrap import setup\n skill_dir, scripts_dir, config_dir = setup()\n\n # Now imports work:\n from utils.logging_utils import get_or_setup_logger\n from core.doc_resolver import DocResolver\n \"\"\"\n scripts_dir = _find_scripts_dir(from_path)\n skill_dir = scripts_dir.parent\n config_dir = skill_dir / 'config'\n\n # Add to sys.path if not already present\n for path_dir in [scripts_dir, config_dir]:\n path_str = str(path_dir)\n if path_str not in sys.path:\n sys.path.insert(0, path_str)\n\n return skill_dir, scripts_dir, config_dir\n\n\n# Auto-setup when imported (for simple `import bootstrap` usage)\n# Scripts can ignore return value if they don't need the paths\n_skill_dir, _scripts_dir, _config_dir = setup()\n\n# Apply dev mode override if CODEX_DOCS_DEV_ROOT is set\n# This allows developers to run scripts from the installed location\n# but have paths resolve to their dev repo for testing\ntry:\n from utils.dev_mode import get_effective_skill_dir\n _effective_skill_dir = get_effective_skill_dir(_skill_dir)\n if _effective_skill_dir != _skill_dir:\n _skill_dir = _effective_skill_dir\n _scripts_dir = _skill_dir / 'scripts'\n _config_dir = _skill_dir / 'config'\n # Update sys.path for the new directories\n for path_dir in [_scripts_dir, _config_dir]:\n path_str = str(path_dir)\n if path_str not in sys.path:\n sys.path.insert(0, path_str)\nexcept ImportError:\n # dev_mode not available during initial setup or in minimal environments\n pass\nexcept ValueError as e:\n # Invalid dev root path - print warning but continue with fallback\n import sys as _sys\n print(f\"Warning: {e}\", file=_sys.stderr)\n\n# Export for scripts that want paths via attribute access\nskill_dir = _skill_dir\nscripts_dir = _scripts_dir\nconfig_dir = _config_dir\n","content_type":"text/x-python; charset=utf-8","language":"python","size":4205,"content_sha256":"8965c1dda141257e122380992160e06c0d507635b549dfd39df085ee7ce4f8d7"},{"filename":"scripts/core/__init__.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCore scripts for OpenAI Codex CLI documentation management.\n\nThis package contains the core functionality:\n- llms_parser: Parse llms.txt and llms-full.txt formats\n- scrape_docs: Fetch and save documentation from sources\n- doc_resolver: Search and resolve documentation\n- find_docs: CLI interface for documentation search\n\"\"\"\n\nfrom pathlib import Path\n\n__all__ = [\n 'llms_parser',\n 'scrape_docs',\n 'doc_resolver',\n 'find_docs',\n]\n","content_type":"text/x-python; charset=utf-8","language":"python","size":492,"content_sha256":"6970c244f6b7c6bb5b1e83931b4fa29a90935e78394763863169b83b09f26c7a"},{"filename":"scripts/core/doc_resolver.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\ndoc_resolver.py - Resolve doc_id and keywords to file paths\n\nAdapted for OpenAI Codex CLI Documentation from google-ecosystem gemini-cli-docs skill.\n\nProvides efficient resolution of documentation references:\n- doc_id -> file path (canonical or extract)\n- keyword search -> doc_ids\n- alias resolution (for renamed docs)\n- Category/tag filtering\n\nUses IndexManager for efficient large file handling.\nUses inverted index caching for O(1) keyword lookups.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n\nimport json\nimport re\nfrom typing import Any, Dict\n\n# Cache directory for inverted index persistence\nCACHE_DIR = Path(__file__).resolve().parent.parent.parent / \".cache\"\nINVERTED_INDEX_CACHE = CACHE_DIR / \"inverted_index.json\"\n\n# Import CacheManager for content hash-based cache validation\ntry:\n from utils.cache_manager import CacheManager\nexcept ImportError:\n CacheManager = None # Fallback to mtime-based validation if unavailable\n\nfrom utils.metadata_utils import normalize_tags\nfrom utils.script_utils import configure_utf8_output, EXIT_INDEX_ERROR, normalize_url_for_display\nfrom utils.search_constants import (\n TITLE, DESCRIPTION, KEYWORD, TAG, IDENTIFIER,\n PENALTIES, COVERAGE, SUBSECTION, POSITIONAL\n)\n\n\ndef _tokenize_text(value: str) -> list[str]:\n \"\"\"Split text into alphanumeric tokens.\"\"\"\n return re.findall(r'[a-z0-9]+', value.lower())\n\n\ndef _keyword_variants(keyword: str) -> set[str]:\n \"\"\"Return normalized variants for a keyword (raw, normalized, singular).\"\"\"\n variants: set[str] = set()\n kw_lower = keyword.lower().strip()\n if not kw_lower:\n return variants\n\n variants.add(kw_lower)\n normalized = re.sub(r'[^a-z0-9]+', '', kw_lower)\n if normalized:\n variants.add(normalized)\n\n if kw_lower.endswith('s') and len(kw_lower) > 3:\n singular = kw_lower.rstrip('s')\n variants.add(singular)\n singular_norm = re.sub(r'[^a-z0-9]+', '', singular)\n if singular_norm:\n variants.add(singular_norm)\n\n return variants\n\n\ndef _build_identifier_tokens(doc_id: str, metadata: dict[str, Any]) -> set[str]:\n \"\"\"Build a set of identifier tokens from doc_id and path/file names.\"\"\"\n tokens: set[str] = set(filter(None, re.split(r'[-_/]+', doc_id.lower())))\n normalized_doc_id = re.sub(r'[^a-z0-9]+', '', doc_id.lower())\n if normalized_doc_id:\n tokens.add(normalized_doc_id)\n\n path_value = metadata.get('path', '')\n if path_value:\n try:\n path_obj = Path(path_value)\n stem = path_obj.stem.lower()\n except Exception:\n stem = str(path_value).lower()\n\n tokens.update(filter(None, re.split(r'[\\\\/_-]+', stem)))\n normalized_stem = re.sub(r'[^a-z0-9]+', '', stem)\n if normalized_stem:\n tokens.add(normalized_stem)\n\n return {token for token in tokens if token}\n\n# Configure UTF-8 output for Windows console compatibility\nconfigure_utf8_output()\n\n# Optional: structured logging for search observability\n# Logging is optional in doc_resolver since it's used as a library\ntry:\n from utils.logging_utils import get_or_setup_logger\n _logger = get_or_setup_logger(__file__, log_category=\"search\")\nexcept ImportError:\n _logger = None\n\ntry:\n from management.index_manager import IndexManager\n from utils.config_helpers import (\n get_domain_weight, get_query_stop_words, get_generic_verbs,\n get_generic_config_terms, get_natural_language_stop_words\n)\n # Import extract_subsection for content extraction\n try:\n from management.extract_subsection import MarkdownExtractor\n except ImportError:\n MarkdownExtractor = None\nexcept ImportError:\n print(\"Error: Could not import helpers or index_manager\")\n print(\"Make sure common_paths.py, index_manager.py, and config_helpers.py are available\")\n sys.exit(EXIT_INDEX_ERROR)\n\n\nclass DocResolver:\n \"\"\"Resolve doc_id and keywords to file paths using a cached index snapshot.\"\"\"\n\n def __init__(self, base_dir: Path):\n \"\"\"\n Initialize resolver\n\n Args:\n base_dir: Base directory containing index.yaml\n \"\"\"\n self.base_dir = Path(base_dir)\n self.index_manager = IndexManager(self.base_dir)\n # Load index once per resolver instance so that search/related lookups\n # and doc_id resolution share the same in-memory snapshot instead of\n # repeatedly re-reading index.yaml.\n self._index: dict[str, Dict] = self.index_manager.load_all() or {}\n self._alias_cache: dict[str, str] = {} # alias -> doc_id cache\n\n # Inverted index for O(1) keyword lookup (built lazily on first search)\n self._inverted_index: dict[str, set[str]] | None = None\n self._tag_index: dict[str, set[str]] | None = None\n self._category_index: dict[str, set[str]] | None = None\n\n # Store index path for cache validation\n self._index_path = self.base_dir / \"index.yaml\"\n\n # Initialize CacheManager for hash-based validation (if available)\n self._cache_manager = CacheManager(self.base_dir) if CacheManager else None\n\n def _get_index_mtime(self) -> float:\n \"\"\"Get modification time of index.yaml.\"\"\"\n try:\n return self._index_path.stat().st_mtime\n except OSError:\n return 0.0\n\n def _is_cache_valid(self) -> bool:\n \"\"\"Check if inverted index cache is valid.\n\n Uses content hash-based validation via CacheManager if available,\n falling back to mtime-based validation otherwise. Hash-based validation\n correctly handles git pull scenarios where mtime changes but content\n may or may not have changed.\n \"\"\"\n # Use CacheManager for robust hash-based validation\n if self._cache_manager:\n return self._cache_manager.is_inverted_index_valid()\n\n # Fallback to mtime-based validation\n if not INVERTED_INDEX_CACHE.exists():\n return False\n try:\n index_mtime = self._get_index_mtime()\n cache_mtime = INVERTED_INDEX_CACHE.stat().st_mtime\n return cache_mtime > index_mtime\n except OSError:\n return False\n\n def _load_cached_index(self) -> bool:\n \"\"\"\n Load inverted index from cache file.\n\n Returns:\n True if cache was loaded successfully, False otherwise.\n \"\"\"\n try:\n with open(INVERTED_INDEX_CACHE, 'r', encoding='utf-8') as f:\n data = json.load(f)\n\n # Convert lists back to sets\n self._inverted_index = {k: set(v) for k, v in data.get('inverted_index', {}).items()}\n self._tag_index = {k: set(v) for k, v in data.get('tag_index', {}).items()}\n self._category_index = {k: set(v) for k, v in data.get('category_index', {}).items()}\n\n if _logger:\n _logger.debug(\n f\"Loaded inverted index from cache: {len(self._inverted_index)} terms, \"\n f\"{len(self._tag_index)} tags, {len(self._category_index)} categories\"\n )\n return True\n except (json.JSONDecodeError, OSError, KeyError) as e:\n if _logger:\n _logger.warning(f\"Failed to load inverted index cache: {e}\")\n # Reset indexes on failure\n self._inverted_index = None\n self._tag_index = None\n self._category_index = None\n return False\n\n def _save_index_cache(self) -> None:\n \"\"\"Save inverted index to cache file for persistence across sessions.\"\"\"\n if self._inverted_index is None:\n return\n\n try:\n # Ensure cache directory exists\n CACHE_DIR.mkdir(parents=True, exist_ok=True)\n\n # Convert sets to lists for JSON serialization\n data = {\n 'inverted_index': {k: sorted(v) for k, v in self._inverted_index.items()},\n 'tag_index': {k: sorted(v) for k, v in self._tag_index.items()},\n 'category_index': {k: sorted(v) for k, v in self._category_index.items()},\n }\n\n # Write atomically (temp file + rename)\n temp_cache = INVERTED_INDEX_CACHE.with_suffix('.tmp')\n with open(temp_cache, 'w', encoding='utf-8') as f:\n json.dump(data, f, separators=(',', ':')) # Compact JSON\n temp_cache.replace(INVERTED_INDEX_CACHE)\n\n # Mark cache as built with CacheManager (stores content hash)\n if self._cache_manager:\n self._cache_manager.mark_inverted_index_built()\n\n if _logger:\n _logger.debug(f\"Saved inverted index cache: {INVERTED_INDEX_CACHE}\")\n except OSError as e:\n if _logger:\n _logger.warning(f\"Failed to save inverted index cache: {e}\")\n\n def resolve_doc_id(self, doc_id: str, extract_path: str | None = None) -> Path | None:\n \"\"\"\n Resolve doc_id to file path\n\n Args:\n doc_id: Document ID to resolve\n extract_path: Optional extract path (for skill extracts)\n\n Returns:\n Path to document, or None if not found\n \"\"\"\n # Check alias cache first\n original_doc_id = doc_id\n if doc_id in self._alias_cache:\n doc_id = self._alias_cache[doc_id]\n\n # Get entry from index\n entry = self._index.get(doc_id)\n if not entry:\n # Try to find by alias (only if we haven't already resolved from cache)\n if doc_id == original_doc_id:\n resolved_doc_id = self._resolve_alias(doc_id)\n if resolved_doc_id and resolved_doc_id != doc_id:\n doc_id = resolved_doc_id\n entry = self.index_manager.get_entry(doc_id)\n\n if not entry:\n return None\n\n # If extract_path specified, return extract path (handle both relative and absolute)\n if extract_path:\n extract_path_obj = Path(extract_path)\n # Resolve relative paths\n if not extract_path_obj.is_absolute():\n extract_path_obj = extract_path_obj.resolve()\n if extract_path_obj.exists():\n return extract_path_obj\n\n # Return canonical path (normalize path separators for cross-platform)\n path_str = entry.get('path')\n if not path_str:\n return None\n\n # Normalize path separators (index stores forward slashes)\n path_str_normalized = str(path_str).replace('\\\\', '/')\n canonical_path = self.base_dir / path_str_normalized\n if canonical_path.exists():\n return canonical_path\n\n return None\n\n def _resolve_alias(self, alias: str) -> str | None:\n \"\"\"\n Resolve alias to doc_id\n\n Args:\n alias: Alias to resolve\n\n Returns:\n doc_id if found, None otherwise\n \"\"\"\n # Check cache first\n if alias in self._alias_cache:\n return self._alias_cache[alias]\n\n # Search through all entries for alias\n for doc_id, metadata in self._index.items():\n aliases = metadata.get('aliases', [])\n if isinstance(aliases, list) and alias in aliases:\n self._alias_cache[alias] = doc_id\n return doc_id\n\n return None\n\n def _build_inverted_index(self) -> None:\n \"\"\"\n Build inverted indexes for O(1) keyword lookup.\n\n Creates:\n - _inverted_index: keyword/variant -> set of doc_ids\n - _tag_index: tag -> set of doc_ids\n - _category_index: category -> set of doc_ids\n\n This is called lazily on first search and cached for subsequent searches.\n Uses file-based caching to persist across script invocations.\n Provides 10-100x speedup by eliminating O(n) full index scans.\n \"\"\"\n if self._inverted_index is not None:\n return # Already built in this session\n\n import time\n start_time = time.time()\n\n # Try to load from disk cache first (much faster than rebuilding)\n if self._is_cache_valid() and self._load_cached_index():\n cache_time = time.time() - start_time\n if _logger:\n _logger.info(f\"Loaded inverted index from cache ({cache_time*1000:.1f}ms)\")\n return\n\n self._inverted_index = {}\n self._tag_index = {}\n self._category_index = {}\n\n for doc_id, metadata in self._index.items():\n # Index keywords and their variants\n doc_keywords = metadata.get('keywords', [])\n if isinstance(doc_keywords, str):\n doc_keywords = [doc_keywords]\n\n for kw in doc_keywords:\n kw_lower = kw.lower().strip()\n if not kw_lower:\n continue\n\n # Add the keyword itself\n if kw_lower not in self._inverted_index:\n self._inverted_index[kw_lower] = set()\n self._inverted_index[kw_lower].add(doc_id)\n\n # Add variants (normalized, singular)\n for variant in _keyword_variants(kw_lower):\n if variant not in self._inverted_index:\n self._inverted_index[variant] = set()\n self._inverted_index[variant].add(doc_id)\n\n # Add tokens from keyword\n for token in _tokenize_text(kw_lower):\n if token not in self._inverted_index:\n self._inverted_index[token] = set()\n self._inverted_index[token].add(doc_id)\n\n # Index title words\n title = metadata.get('title', '').lower()\n for token in _tokenize_text(title):\n if token not in self._inverted_index:\n self._inverted_index[token] = set()\n self._inverted_index[token].add(doc_id)\n\n # Index description words\n description = metadata.get('description', '').lower()\n for token in _tokenize_text(description):\n if token not in self._inverted_index:\n self._inverted_index[token] = set()\n self._inverted_index[token].add(doc_id)\n\n # Index tags\n doc_tags = metadata.get('tags', [])\n if isinstance(doc_tags, str):\n doc_tags = [doc_tags]\n for tag in doc_tags:\n tag_lower = tag.lower().strip()\n if tag_lower:\n if tag_lower not in self._tag_index:\n self._tag_index[tag_lower] = set()\n self._tag_index[tag_lower].add(doc_id)\n\n # Index category\n category = metadata.get('category', '').lower().strip()\n if category:\n if category not in self._category_index:\n self._category_index[category] = set()\n self._category_index[category].add(doc_id)\n\n # Index doc_id tokens (for identifier matching)\n for token in _tokenize_text(doc_id):\n if token not in self._inverted_index:\n self._inverted_index[token] = set()\n self._inverted_index[token].add(doc_id)\n\n # Log inverted index build stats\n build_time = time.time() - start_time\n if _logger:\n _logger.info(\n f\"Inverted index built: {len(self._inverted_index)} terms, \"\n f\"{len(self._tag_index)} tags, {len(self._category_index)} categories \"\n f\"({build_time*1000:.1f}ms)\"\n )\n\n # Save to cache for future invocations\n self._save_index_cache()\n\n def _score_subsection_matches(\n self,\n subsections: list[dict],\n keyword_lower: list[str],\n has_substantive_match: bool,\n main_content_matches: int\n ) -> tuple[dict | None, int]:\n \"\"\"\n Score subsection matches and find the best matching subsection.\n\n Args:\n subsections: List of subsection dicts from document metadata\n keyword_lower: Lowercased search keywords\n has_substantive_match: Whether any substantive (non-generic) keyword matched\n main_content_matches: Count of main content matches\n\n Returns:\n Tuple of (best_subsection dict or None, subsection_bonus score)\n \"\"\"\n best_subsection = None\n best_subsection_score = 0\n subsection_bonus = 0\n\n for subsection in subsections:\n if not isinstance(subsection, dict):\n continue\n\n subsection_score = 0\n anchor = subsection.get('anchor', '')\n heading = subsection.get('heading', '').lower()\n\n # Score heading matches\n if all(kw in heading for kw in keyword_lower):\n subsection_score += SUBSECTION.all_kw_in_heading\n if has_substantive_match:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_all_substantive)\n elif main_content_matches > 0:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_all_main_content)\n else:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_all_default)\n else:\n for kw in keyword_lower:\n if kw in heading:\n subsection_score += SUBSECTION.single_kw_in_heading\n if has_substantive_match:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_single_substantive)\n elif main_content_matches > 0:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_single_main_content)\n else:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_single_default)\n\n # Score subsection keyword matches\n sub_keywords = subsection.get('keywords', [])\n if isinstance(sub_keywords, list):\n sub_keywords_lower = [k.lower() for k in sub_keywords]\n if all(any(kw in skw or skw in kw for skw in sub_keywords_lower) for kw in keyword_lower):\n subsection_score += SUBSECTION.all_kw_in_keywords\n if has_substantive_match:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_all_substantive)\n elif main_content_matches > 0:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_all_main_content)\n else:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_all_default)\n else:\n for kw in keyword_lower:\n if kw in sub_keywords_lower:\n subsection_score += SUBSECTION.single_kw_in_keywords\n if has_substantive_match:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_single_substantive)\n elif main_content_matches > 0:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_single_main_content)\n else:\n subsection_bonus = max(subsection_bonus, SUBSECTION.bonus_kw_single_default)\n\n # Track best subsection\n if subsection_score > best_subsection_score:\n best_subsection_score = subsection_score\n best_subsection = {\n 'anchor': anchor,\n 'heading': subsection.get('heading', ''),\n 'score': subsection_score\n }\n\n return best_subsection, subsection_bonus\n\n def _get_candidate_doc_ids(self, keywords: list[str], category: str | None = None,\n tags: list[str] | None = None) -> set[str]:\n \"\"\"\n Get candidate doc_ids using inverted index for O(1) lookup.\n\n Args:\n keywords: List of search keywords\n category: Optional category filter\n tags: Optional tag filter\n\n Returns:\n Set of doc_ids that might match the search query\n \"\"\"\n self._build_inverted_index()\n\n if not self._inverted_index:\n return set(self._index.keys()) # Fallback to full scan\n\n candidates = None\n\n # For each keyword, get matching doc_ids\n for kw in keywords:\n kw_lower = kw.lower().strip()\n if not kw_lower:\n continue\n\n # Collect all doc_ids matching this keyword or its variants/tokens\n kw_matches = set()\n\n # Direct match\n if kw_lower in self._inverted_index:\n kw_matches.update(self._inverted_index[kw_lower])\n\n # Variant matches\n for variant in _keyword_variants(kw_lower):\n if variant in self._inverted_index:\n kw_matches.update(self._inverted_index[variant])\n\n # Token matches (for compound terms)\n # Only skip tokenization for true file-like terms (end with extension)\n # This allows \"claude.md memory\" to be tokenized while keeping \"claude.md\" atomic\n is_file_like = bool(re.match(r'^[a-z0-9_.-]+\\.[a-z]{1,4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, kw_lower))\n if not is_file_like:\n for token in _tokenize_text(kw_lower):\n if token in self._inverted_index:\n kw_matches.update(self._inverted_index[token])\n\n # Union all matches for this keyword\n if candidates is None:\n candidates = kw_matches\n else:\n # Use union to allow partial matches (any keyword match)\n candidates.update(kw_matches)\n\n if candidates is None:\n candidates = set()\n\n # Apply category filter\n if category and self._category_index:\n category_lower = category.lower().strip()\n if category_lower in self._category_index:\n candidates &= self._category_index[category_lower]\n else:\n candidates = set() # Category doesn't exist\n\n # Apply tag filter\n if tags and self._tag_index:\n tag_matches = set()\n for tag in tags:\n if tag:\n tag_lower = tag.lower().strip()\n if tag_lower in self._tag_index:\n tag_matches.update(self._tag_index[tag_lower])\n if tag_matches:\n candidates &= tag_matches\n else:\n candidates = set() # No matching tags\n\n return candidates\n\n def search_by_keyword(self, keywords: list[str], category: str | None = None,\n tags: list[str | None] = None, limit: int = 10,\n return_scores: bool = False) -> list[tuple[str, Dict]]:\n \"\"\"\n Search documents by keywords\n\n Args:\n keywords: List of keywords to search for\n category: Optional category filter\n tags: Optional tags filter\n limit: Maximum number of results\n return_scores: If True, include score in metadata['_score']\n\n Returns:\n List of (doc_id, metadata) tuples, sorted by relevance\n \"\"\"\n import time\n search_start = time.time()\n\n results = []\n keyword_lower = [\n k.lower().strip() for k in keywords\n if isinstance(k, str) and k.strip()\n ]\n if not keyword_lower:\n return results\n\n # Filter out query stop words (e.g., \"when\", \"how\", \"to\", \"best\", \"practices\")\n # This transforms queries like \"when to use subagents\" -> [\"subagents\"]\n query_stop_words = get_query_stop_words()\n original_keywords = keyword_lower.copy()\n keyword_lower = [kw for kw in keyword_lower if kw not in query_stop_words]\n\n # If ALL keywords were filtered, fall back to original (rare edge case)\n if not keyword_lower and original_keywords:\n keyword_lower = original_keywords\n if _logger:\n _logger.debug(f\"All keywords were stop words, using original: {original_keywords}\")\n\n keyword_variants = {kw: _keyword_variants(kw) for kw in keyword_lower}\n # For file-like terms (containing '.'), don't tokenize - treat as atomic\n # This prevents 'claude.md' from matching via 'claude' token in doc_ids\n keyword_tokens = {\n kw: set() if '.' in kw else (set(_tokenize_text(kw)) or {kw})\n for kw in keyword_lower\n }\n\n # Load generic verbs from config (with fallback defaults)\n generic_verbs = get_generic_verbs() or {'create', 'use', 'get', 'add', 'remove', 'update', 'delete', 'set', 'configure', 'run', 'build', 'make', 'do'}\n\n # Use inverted index to get candidate doc_ids (O(1) lookup vs O(n) full scan)\n # This provides 10-100x speedup for typical queries\n candidate_doc_ids = self._get_candidate_doc_ids(keyword_lower, category, tags)\n\n for doc_id in candidate_doc_ids:\n metadata = self._index.get(doc_id)\n if not metadata:\n continue\n\n # Category/tag filters already applied by _get_candidate_doc_ids\n # But we still need the tag set for scoring\n doc_tags = metadata.get('tags', [])\n if isinstance(doc_tags, str):\n doc_tags = [doc_tags]\n doc_tags_lower = [t.lower() for t in doc_tags]\n doc_tag_set = set(doc_tags_lower)\n\n doc_keywords = metadata.get('keywords', [])\n if isinstance(doc_keywords, str):\n doc_keywords = [doc_keywords]\n doc_keywords_lower = [k.lower() for k in doc_keywords]\n # Include index for positional scoring (earlier keywords = higher priority)\n doc_keywords_data = [\n {\n 'value': doc_kw,\n 'variants': _keyword_variants(doc_kw),\n 'tokens': set(_tokenize_text(doc_kw)),\n 'index': idx # Position in keyword list (0 = first/highest priority)\n }\n for idx, doc_kw in enumerate(doc_keywords_lower)\n ]\n\n doc_identifier_tokens = _build_identifier_tokens(doc_id, metadata)\n\n score = 0\n main_content_matches = 0\n matched_keywords = set()\n best_keyword_position = len(doc_keywords) # Track best (lowest) position of matched keyword\n\n title = metadata.get('title', '').lower()\n description = metadata.get('description', '').lower()\n path = metadata.get('path', '').lower()\n url = metadata.get('url', '').lower()\n\n for kw in keyword_lower:\n kw_variants = keyword_variants.get(kw, {kw})\n kw_token_set = keyword_tokens.get(kw, {kw})\n\n if kw in title:\n score += TITLE.exact_match\n main_content_matches += 1\n matched_keywords.add(kw)\n elif re.search(r'\\b' + re.escape(kw) + r'\\b', title):\n score += TITLE.word_boundary_match\n main_content_matches += 1\n matched_keywords.add(kw)\n\n if kw in description:\n score += DESCRIPTION.exact_match\n main_content_matches += 1\n matched_keywords.add(kw)\n elif re.search(r'\\b' + re.escape(kw) + r'\\b', description):\n score += DESCRIPTION.word_boundary_match\n main_content_matches += 1\n matched_keywords.add(kw)\n\n for doc_kw in doc_keywords_data:\n matched = False\n if kw_variants & doc_kw['variants']:\n score += KEYWORD.variant_match\n main_content_matches += 1\n matched_keywords.add(kw)\n matched = True\n elif kw_token_set & doc_kw['tokens']:\n score += KEYWORD.token_match\n main_content_matches += 1\n matched_keywords.add(kw)\n matched = True\n elif kw in doc_kw['value'] or doc_kw['value'] in kw:\n score += KEYWORD.substring_match\n main_content_matches += 1\n matched_keywords.add(kw)\n matched = True\n\n if matched:\n # Track best keyword position for tiebreaking (lower = better)\n best_keyword_position = min(best_keyword_position, doc_kw['index'])\n break\n\n if kw in doc_tag_set:\n score += TAG.exact_match\n main_content_matches += 1\n matched_keywords.add(kw)\n elif any(variant in doc_tag_set for variant in kw_variants):\n score += TAG.variant_match\n main_content_matches += 1\n matched_keywords.add(kw)\n\n if any(variant in doc_identifier_tokens for variant in kw_variants):\n score += IDENTIFIER.identifier_match\n main_content_matches += 1\n matched_keywords.add(kw)\n\n if kw in path or kw in url:\n score += IDENTIFIER.path_url_match\n\n substantive_keywords = [kw for kw in keyword_lower if kw not in generic_verbs]\n matched_substantive = [kw for kw in substantive_keywords if kw in matched_keywords]\n has_substantive_match = len(matched_substantive) > 0\n\n # Apply penalty for generic term matches\n # Generic terms like \"configuration\", \"setup\", \"installation\" are too broad\n # and cause ranking collapse when mixed with specific terms\n # Loaded from filtering.yaml for centralized configuration\n generic_config_terms = get_generic_config_terms()\n\n # Identify which matched keywords are generic vs specific\n generic_matches = [kw for kw in matched_keywords if kw in generic_config_terms]\n specific_matches = [kw for kw in matched_keywords if kw not in generic_config_terms]\n\n # If doc ONLY matches generic terms (no specific term matches), severely penalize\n if generic_matches and not specific_matches:\n # Only generic terms matched - reduce score to near zero\n # This effectively filters out generic docs from appearing in results\n score *= PENALTIES.only_generic\n elif generic_matches and matched_keywords:\n # Mix of generic and specific - apply proportional penalty\n generic_ratio = len(generic_matches) / len(matched_keywords)\n if generic_ratio >= PENALTIES.high_threshold:\n # High generic ratio: reduce score significantly\n score *= PENALTIES.high_ratio\n elif generic_ratio >= PENALTIES.medium_threshold:\n # Medium generic ratio: reduce score moderately\n score *= PENALTIES.medium_ratio\n\n # Calculate query term coverage (what percentage of query terms matched)\n # Boost docs that match ALL query terms over docs that match only SOME\n num_query_terms = len(keyword_lower)\n num_matched_terms = len(matched_keywords)\n term_coverage = num_matched_terms / num_query_terms if num_query_terms > 0 else 0\n\n # Apply coverage multiplier to score\n # Full coverage (100%) = 1.5x to 2.0x multiplier (depending on title/heading match)\n # Partial coverage (50%) = 1.0x multiplier (no change)\n # This ensures docs matching all terms rank higher\n if term_coverage >= 1.0:\n # All terms matched - check if they ALL appear in title or heading\n all_in_title = all(kw in title for kw in keyword_lower)\n all_in_description = all(kw in description for kw in keyword_lower)\n\n if all_in_title or all_in_description:\n # ALL query terms in title/description - strongest signal\n coverage_multiplier = COVERAGE.all_in_title\n else:\n # All terms matched across metadata - significant boost\n coverage_multiplier = COVERAGE.all_terms\n elif term_coverage >= COVERAGE.most_threshold:\n # Most terms matched - moderate boost\n coverage_multiplier = COVERAGE.most_terms\n else:\n # Partial match - no boost\n coverage_multiplier = COVERAGE.partial\n\n score *= coverage_multiplier\n\n # Score subsection matches (extracted method for readability)\n subsections = metadata.get('subsections', [])\n best_subsection, subsection_bonus = (None, 0)\n if subsections and isinstance(subsections, list):\n best_subsection, subsection_bonus = self._score_subsection_matches(\n subsections, keyword_lower, has_substantive_match, main_content_matches\n )\n\n score += subsection_bonus\n matched_subsection_anchor = best_subsection['anchor'] if best_subsection else None\n\n if main_content_matches == 0:\n continue\n\n domain = metadata.get('domain', '')\n doc_category_for_weight = metadata.get('category', '')\n domain_key = domain\n if domain == 'developers.openai.com' and doc_category_for_weight:\n domain_key = f'developers.openai.com/{doc_category_for_weight}'\n domain_weight = get_domain_weight(domain_key)\n score *= domain_weight\n\n if score > 0:\n # Add positional bonus for tiebreaking (earlier keyword position = higher bonus)\n # Bonus is small to avoid affecting main ranking, but enough to break ties\n # Formula: max_bonus / (position + 1) - so position 0 gets max, position 1 gets half, etc.\n if best_keyword_position \u003c len(doc_keywords):\n positional_bonus = POSITIONAL.max_bonus / (best_keyword_position + 1)\n score += positional_bonus\n\n result_metadata = metadata.copy()\n\n if matched_subsection_anchor:\n original_url = result_metadata.get('url', '')\n if original_url and matched_subsection_anchor:\n normalized_url = normalize_url_for_display(original_url)\n if '#' not in normalized_url:\n result_metadata['url'] = normalized_url + matched_subsection_anchor\n else:\n result_metadata['url'] = normalized_url\n if best_subsection:\n result_metadata['_matched_subsection'] = {\n 'anchor': matched_subsection_anchor,\n 'heading': best_subsection.get('heading'),\n 'score': best_subsection.get('score', 0)\n }\n result_metadata['_subsection_hint'] = True\n result_metadata['_extraction_command'] = (\n f\"python scripts/get_subsection_content.py {doc_id} \"\n f\"--section \\\"{best_subsection.get('heading')}\\\"\"\n )\n\n results.append((score, best_keyword_position, doc_id, result_metadata))\n\n # Sort by score (desc), then by keyword position (asc) for tiebreaking\n results.sort(key=lambda x: (-x[0], x[1]))\n final_results = []\n for score, kw_position, doc_id, metadata in results[:limit]:\n if return_scores:\n metadata['_score'] = round(score, 2)\n final_results.append((doc_id, metadata))\n\n # Log search performance\n search_time = time.time() - search_start\n if _logger:\n top_result = final_results[0][0] if final_results else \"none\"\n _logger.debug(\n f\"Search: keywords={keyword_lower}, results={len(final_results)}, \"\n f\"candidates={len(results)}, top={top_result}, time={search_time*1000:.1f}ms\"\n )\n\n return final_results\n\n def search_by_natural_language(self, query: str, limit: int = 10,\n return_scores: bool = False) -> list[tuple[str, Dict]]:\n \"\"\"\n Search documents using natural language query\n\n Args:\n query: Natural language search query\n limit: Maximum number of results\n return_scores: If True, include score in metadata['_score']\n\n Returns:\n List of (doc_id, metadata) tuples, sorted by relevance\n \"\"\"\n # Extract keywords from query\n # Remove common stop words (loaded from config for centralized management)\n stop_words = get_natural_language_stop_words()\n\n query_lower = query.lower()\n words = re.findall(r'\\b[a-z]{3,}\\b', query_lower) # Min 3 chars to filter noise like 'md', 'id'\n keywords = [w for w in words if w not in stop_words]\n\n # Include file-like tokens (e.g., claude.md) so we can match documentation filenames\n file_terms = re.findall(r'\\b[a-z0-9]+(?:\\.[a-z0-9]+)+\\b', query_lower)\n for term in file_terms:\n if term not in keywords:\n keywords.append(term)\n\n if not keywords:\n return []\n\n return self.search_by_keyword(keywords, limit=limit, return_scores=return_scores)\n\n def get_by_category(self, category: str) -> list[tuple[str, Dict]]:\n \"\"\"Get all documents in a category\"\"\"\n results = []\n for doc_id, metadata in self.index_manager.list_entries():\n doc_category = metadata.get('category', '').lower()\n if doc_category == category.lower():\n results.append((doc_id, metadata))\n return results\n\n def get_by_tag(self, tag: str) -> list[tuple[str, Dict]]:\n \"\"\"Get all documents with a specific tag\"\"\"\n results = []\n tag_lower = tag.lower().strip()\n for doc_id, metadata in self._index.items():\n doc_tags_lower = normalize_tags(metadata.get('tags', []))\n if tag_lower in doc_tags_lower:\n results.append((doc_id, metadata))\n return results\n\n def get_related_docs(self, doc_id: str, limit: int = 5) -> list[tuple[str, Dict]]:\n \"\"\"\n Find related documents based on shared keywords/tags\n\n Args:\n doc_id: Document ID to find related docs for\n limit: Maximum number of results\n\n Returns:\n List of (doc_id, metadata) tuples\n \"\"\"\n entry = self.index_manager.get_entry(doc_id)\n if not entry:\n return []\n\n # Get keywords and tags from source doc\n keywords = entry.get('keywords', [])\n if isinstance(keywords, str):\n keywords = [keywords]\n tags = entry.get('tags', [])\n if isinstance(tags, str):\n tags = [tags]\n\n if not keywords and not tags:\n return []\n\n # Search for docs with shared keywords/tags\n results = []\n for other_id, other_meta in self._index.items():\n if other_id == doc_id:\n continue\n\n score = 0\n\n # Check shared keywords\n other_keywords = other_meta.get('keywords', [])\n if isinstance(other_keywords, str):\n other_keywords = [other_keywords]\n shared_keywords = set(k.lower() for k in keywords) & set(k.lower() for k in other_keywords)\n score += len(shared_keywords) * 2\n\n # Check shared tags\n other_tags = other_meta.get('tags', [])\n if isinstance(other_tags, str):\n other_tags = [other_tags]\n shared_tags = set(t.lower() for t in tags) & set(t.lower() for t in other_tags)\n score += len(shared_tags) * 3\n\n if score > 0:\n results.append((score, other_id, other_meta))\n\n # Sort by score and return top results\n results.sort(key=lambda x: x[0], reverse=True)\n return [(doc_id, metadata) for _, doc_id, metadata in results[:limit]]\n\n def get_content(self, doc_id: str, section: str | None = None) -> dict[str, Any | None]:\n \"\"\"\n Get document content (full or partial section).\n\n Args:\n doc_id: Document identifier\n section: Optional section heading to extract (if None, returns full content)\n\n Returns:\n Dictionary with keys:\n - content: Markdown content (partial or full)\n - content_type: \"partial\" | \"full\" | \"link\"\n - section_ref: Hashtag reference if partial (e.g., \"#installation\")\n - doc_id: Document identifier\n - url: Source URL if available\n - title: Document title\n - description: Document description\n - warning: Warning about not storing file paths\n\n Returns None if doc_id not found or file doesn't exist.\n \"\"\"\n # Resolve doc_id to file path\n path = self.resolve_doc_id(doc_id)\n if not path or not path.exists():\n return None\n\n # Get metadata from index\n entry = self.index_manager.get_entry(doc_id)\n if not entry:\n return None\n\n # Generate hashtag reference from section heading\n section_ref = None\n if section:\n # Convert section heading to hashtag format (lowercase, replace spaces with hyphens)\n section_ref = '#' + re.sub(r'[^\\w\\s-]', '', section.lower()).strip().replace(' ', '-')\n\n # Extract content\n content = None\n content_type = \"link\"\n\n if MarkdownExtractor is None:\n # Fallback: return link only if extractor not available\n return {\n 'content': None,\n 'content_type': 'link',\n 'section_ref': section_ref,\n 'doc_id': doc_id,\n 'url': normalize_url_for_display(entry.get('url')),\n 'title': entry.get('title'),\n 'description': entry.get('description'),\n 'warning': 'Warning: Do not store file paths. Use doc_id references or invoke codex-cli-docs skill for access.'\n }\n\n try:\n extractor = MarkdownExtractor(path)\n\n if section:\n # Extract specific section\n content = extractor.extract_section(section)\n if content:\n content_type = \"partial\"\n else:\n # Section not found, return full content as fallback\n content = extractor.body\n content_type = \"full\"\n else:\n # Return full content\n content = extractor.body\n content_type = \"full\"\n except Exception:\n # If extraction fails, return link only\n content = None\n content_type = \"link\"\n\n return {\n 'content': content,\n 'content_type': content_type,\n 'section_ref': section_ref,\n 'doc_id': doc_id,\n 'url': normalize_url_for_display(entry.get('url')),\n 'title': entry.get('title'),\n 'description': entry.get('description'),\n 'warning': 'Warning: Do not store file paths. Use doc_id references or invoke codex-cli-docs skill for access.'\n }\n\n\nif __name__ == '__main__':\n # Simple CLI wrapper for DocResolver\n import argparse\n\n parser = argparse.ArgumentParser(\n description='Resolve doc_id to path and optionally show index metadata',\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Resolve a document ID to its markdown path\n python doc_resolver.py developers-openai-com-codex-overview\n\n # Resolve and show metadata\n python doc_resolver.py developers-openai-com-codex-overview --show-metadata\n\"\"\",\n )\n from utils.cli_utils import add_base_dir_argument, resolve_base_dir_from_args\n add_base_dir_argument(parser)\n parser.add_argument(\n '--show-metadata',\n action='store_true',\n help='Also print the full metadata entry from index.yaml',\n )\n parser.add_argument('doc_id', nargs='?', help='Document ID to resolve')\n\n args = parser.parse_args()\n\n # Resolve base directory using cli_utils helper\n base_dir = resolve_base_dir_from_args(args)\n\n resolver = DocResolver(base_dir)\n\n if not args.doc_id:\n print(\"Usage: python doc_resolver.py \u003cdoc_id> [--show-metadata]\")\n else:\n path = resolver.resolve_doc_id(args.doc_id)\n if path:\n print(f\"Resolved: {path}\")\n if args.show_metadata:\n entry = resolver.index_manager.get_entry(args.doc_id)\n if entry is None:\n print(\" (No metadata entry found in index.yaml)\")\n else:\n print(\"Metadata:\")\n for key, value in entry.items():\n print(f\" {key}: {value}\")\n else:\n print(f\"Not found: {args.doc_id}\")\n","content_type":"text/x-python; charset=utf-8","language":"python","size":45944,"content_sha256":"676db90c8792171894843081c6808ee8df079d55f5f728c4f4226aba13d04829"},{"filename":"scripts/core/find_docs.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nfind_docs.py - Find and resolve documentation references\n\nAdapted for OpenAI Codex CLI Documentation from google-ecosystem gemini-cli-docs skill.\n\nCLI tool for discovering and resolving documentation:\n- Resolve doc_id to file path\n- Search by keywords\n- Search by natural language query\n- Filter by category/tag\n- Find related documents\n\nUsage:\n python find_docs.py resolve \u003cdoc_id>\n python find_docs.py search \u003ckeyword1> [keyword2 ...]\n python find_docs.py query \"natural language query\"\n python find_docs.py category \u003ccategory>\n python find_docs.py tag \u003ctag>\n python find_docs.py related \u003cdoc_id>\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n\nimport argparse\nimport json\n\nfrom utils.cli_utils import add_common_index_args\nfrom utils.script_utils import (\n configure_utf8_output,\n resolve_base_dir,\n EXIT_SUCCESS,\n EXIT_NO_RESULTS,\n EXIT_BAD_ARGS,\n EXIT_INDEX_ERROR,\n normalize_url_for_display,\n)\nfrom utils.logging_utils import get_or_setup_logger\n\n# Configure UTF-8 output for Windows console compatibility\nconfigure_utf8_output()\n\n# Script logger (structured, with performance tracking)\nlogger = get_or_setup_logger(__file__, log_category=\"search\")\n\ntry:\n from core.doc_resolver import DocResolver\nexcept ImportError:\n try:\n from scripts.core.doc_resolver import DocResolver\n except ImportError:\n print(\"Error: Could not import doc_resolver\")\n print(\"Make sure doc_resolver.py is available (core/doc_resolver.py).\")\n sys.exit(EXIT_INDEX_ERROR)\n\n\ndef cmd_resolve(resolver: DocResolver, doc_id: str, extract_path: str | None = None, json_output: bool = False) -> int:\n \"\"\"Resolve doc_id to file path. Returns 1 if found, 0 if not found.\"\"\"\n path = resolver.resolve_doc_id(doc_id, extract_path)\n\n if json_output:\n result = {\n 'doc_id': doc_id,\n 'path': str(path) if path else None,\n 'found': path is not None\n }\n print(json.dumps(result, indent=2))\n else:\n if path:\n print(f\"Resolved: {path}\")\n print(f\" doc_id: {doc_id}\")\n else:\n print(f\"Not found: {doc_id}\")\n sys.exit(EXIT_NO_RESULTS)\n\n return 1 if path else 0\n\n\ndef _format_result_entry(doc_id: str, metadata: dict) -> dict:\n \"\"\"\n Format a single result entry with standardized field ordering and classification.\n\n Result Classification:\n - 'subsection': Document has relevant subsection match\n - 'general': Document matches query (full document match)\n\n Priority ordering:\n 1. doc_id (primary identifier for Claude Code references)\n 2. path (local file path - PRIMARY reference for Claude Code)\n 3. section_ref (subsection anchor if applicable)\n 4. section_heading (human-readable section title)\n 5. title (document title)\n 6. url (web URL - SECONDARY/informational only)\n 7. type (subsection or general)\n 8. description, category, tags (metadata)\n 9. extraction_command (if subsection match)\n \"\"\"\n # Build result with proper field ordering\n result = {\n 'doc_id': doc_id,\n 'path': metadata.get('path')\n }\n\n # Get matched subsection (if any)\n matched_subsection = metadata.get('_matched_subsection')\n\n # Classify result type\n if matched_subsection:\n # Document has relevant subsection\n result['type'] = 'subsection'\n result['section_ref'] = matched_subsection.get('anchor')\n result['section_heading'] = matched_subsection.get('heading')\n else:\n # General match (full document match)\n result['type'] = 'general'\n\n # Add remaining fields\n result['title'] = metadata.get('title')\n result['url'] = normalize_url_for_display(metadata.get('url'))\n result['description'] = metadata.get('description')\n result['category'] = metadata.get('category')\n result['tags'] = metadata.get('tags', [])\n\n # Add extraction command for subsections\n if matched_subsection and metadata.get('_extraction_command'):\n result['extraction_command'] = metadata.get('_extraction_command')\n\n return result\n\n\ndef _display_search_results(results: list[tuple[str, dict]], header: str, verbose: bool = False) -> None:\n \"\"\"Display search results with consistent formatting.\n\n Args:\n results: List of (doc_id, metadata) tuples\n header: Header text to display (e.g., \"Found X document(s):\")\n verbose: If True, show score details\n \"\"\"\n print(f\"{header}\\n\")\n for i, (doc_id, metadata) in enumerate(results, 1):\n entry = _format_result_entry(doc_id, metadata)\n\n # Display with clear hierarchy\n type_indicator = \" [SUBSECTION]\" if entry['type'] == 'subsection' else \"\"\n score_indicator = f\" (score: {metadata.get('_score', 'N/A')})\" if verbose else \"\"\n print(f\"{i}. {entry['title']}{type_indicator}{score_indicator}\")\n print(f\" doc_id: {entry['doc_id']}\")\n if entry['path']:\n print(f\" path: {entry['path']}\")\n if entry.get('section_ref'):\n print(f\" section: {entry['section_ref']} ({entry.get('section_heading')})\")\n if entry['url']:\n print(f\" url: {entry['url']} (web reference only)\")\n if entry.get('description'):\n desc = entry['description'][:100] + '...' if len(entry['description']) > 100 else entry['description']\n print(f\" description: {desc}\")\n if entry.get('extraction_command'):\n print(f\" extract: {entry['extraction_command']}\")\n print()\n\n\ndef cmd_search(resolver: DocResolver, keywords: list[str], category: str | None = None,\n tags: list[str | None] = None, limit: int = 10, json_output: bool = False,\n verbose: bool = False) -> int:\n \"\"\"Search documents by keywords. Returns number of results found.\"\"\"\n results = resolver.search_by_keyword(keywords, category=category, tags=tags, limit=limit, return_scores=verbose)\n\n if json_output:\n output = [_format_result_entry(doc_id, metadata) for doc_id, metadata in results]\n print(json.dumps(output, indent=2))\n else:\n if not results:\n print(f\"No documents found for keywords: {', '.join(keywords)}\")\n sys.exit(EXIT_NO_RESULTS)\n\n _display_search_results(results, f\"Found {len(results)} document(s):\", verbose)\n\n return len(results)\n\n\ndef cmd_query(resolver: DocResolver, query: str, limit: int = 10, json_output: bool = False,\n verbose: bool = False) -> int:\n \"\"\"Search documents using natural language query. Returns number of results found.\"\"\"\n results = resolver.search_by_natural_language(query, limit=limit, return_scores=verbose)\n\n if json_output:\n output = [_format_result_entry(doc_id, metadata) for doc_id, metadata in results]\n print(json.dumps(output, indent=2))\n else:\n if not results:\n print(f\"No documents found for query: {query}\")\n sys.exit(EXIT_NO_RESULTS)\n\n _display_search_results(results, f\"Found {len(results)} document(s) for query: '{query}'\", verbose)\n\n return len(results)\n\n\ndef cmd_category(resolver: DocResolver, category: str, json_output: bool = False) -> int:\n \"\"\"List all documents in a category. Returns number of results found.\"\"\"\n results = resolver.get_by_category(category)\n\n if json_output:\n output = [_format_result_entry(doc_id, metadata) for doc_id, metadata in results]\n print(json.dumps(output, indent=2))\n else:\n if not results:\n print(f\"No documents found in category: {category}\")\n sys.exit(EXIT_NO_RESULTS)\n\n print(f\"Documents in category '{category}' ({len(results)}):\\n\")\n for i, (doc_id, metadata) in enumerate(results, 1):\n title = metadata.get('title', 'Untitled')\n print(f\"{i}. {title} ({doc_id})\")\n print()\n\n return len(results)\n\n\ndef cmd_tag(resolver: DocResolver, tag: str, json_output: bool = False) -> int:\n \"\"\"List all documents with a specific tag. Returns number of results found.\"\"\"\n results = resolver.get_by_tag(tag)\n\n if json_output:\n output = [_format_result_entry(doc_id, metadata) for doc_id, metadata in results]\n print(json.dumps(output, indent=2))\n else:\n if not results:\n print(f\"No documents found with tag: {tag}\")\n sys.exit(EXIT_NO_RESULTS)\n\n print(f\"Documents with tag '{tag}' ({len(results)}):\\n\")\n for i, (doc_id, metadata) in enumerate(results, 1):\n title = metadata.get('title', 'Untitled')\n print(f\"{i}. {title} ({doc_id})\")\n print()\n\n return len(results)\n\n\ndef cmd_content(resolver: DocResolver, doc_id: str, section: str | None = None, json_output: bool = False) -> int:\n \"\"\"Get document content (full or partial section). Returns 1 if found, 0 if not found.\"\"\"\n content_result = resolver.get_content(doc_id, section)\n\n if not content_result:\n print(f\"Document not found or content unavailable: {doc_id}\")\n sys.exit(EXIT_NO_RESULTS)\n\n if json_output:\n # Normalize URL in JSON output\n content_result_copy = content_result.copy()\n if 'url' in content_result_copy:\n content_result_copy['url'] = normalize_url_for_display(content_result_copy['url'])\n print(json.dumps(content_result_copy, indent=2))\n else:\n print(f\"Document: {content_result.get('title', doc_id)}\")\n print(f\" doc_id: {doc_id}\")\n if content_result.get('url'):\n print(f\" url: {normalize_url_for_display(content_result.get('url'))}\")\n if content_result.get('section_ref'):\n print(f\" section: {content_result.get('section_ref')}\")\n print(f\" content_type: {content_result.get('content_type', 'unknown')}\")\n print()\n print(\"Warning: \" + content_result.get('warning', 'Do not store file paths.'))\n print()\n if content_result.get('content'):\n content = content_result['content']\n # Show first 500 chars if content is long\n if len(content) > 500:\n print(content[:500] + \"\\n... (truncated, use --json for full content)\")\n else:\n print(content)\n else:\n print(\"(Content not available - link only)\")\n\n return 1 if content_result else 0\n\n\ndef cmd_related(resolver: DocResolver, doc_id: str, limit: int = 5, json_output: bool = False) -> int:\n \"\"\"Find related documents. Returns number of results found.\"\"\"\n results = resolver.get_related_docs(doc_id, limit=limit)\n\n if json_output:\n output = []\n for doc_id_result, metadata in results:\n output.append({\n 'doc_id': doc_id_result,\n 'title': metadata.get('title'),\n 'url': normalize_url_for_display(metadata.get('url'))\n })\n print(json.dumps(output, indent=2))\n else:\n if not results:\n print(f\"No related documents found for: {doc_id}\")\n sys.exit(EXIT_NO_RESULTS)\n\n print(f\"Related documents for '{doc_id}' ({len(results)}):\\n\")\n for i, (related_id, metadata) in enumerate(results, 1):\n title = metadata.get('title', 'Untitled')\n print(f\"{i}. {title} ({related_id})\")\n print()\n\n return len(results)\n\n\ndef main():\n \"\"\"Main entry point\"\"\"\n parser = argparse.ArgumentParser(\n description='Find and resolve documentation references',\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Resolve doc_id to path (using default base directory)\n python find_docs.py resolve developers-openai-com-codex-overview\n\n # Resolve doc_id and output JSON (for tools/agents)\n python find_docs.py --json resolve developers-openai-com-codex-overview\n\n # Get full document content\n python find_docs.py content developers-openai-com-codex-overview\n\n # Get specific section content\n python find_docs.py content developers-openai-com-codex-overview --section \"Installation\"\n\n # Search by keywords\n python find_docs.py search sandbox configuration\n\n # Natural language search\n python find_docs.py query \"how to configure sandbox mode\"\n\n # List by category\n python find_docs.py category api\n\n # List by tag\n python find_docs.py tag sandbox\n\n # Find related docs\n python find_docs.py related developers-openai-com-codex-overview\n \"\"\"\n )\n\n add_common_index_args(parser, include_json=True)\n parser.add_argument('--limit', type=int, default=10, help='Maximum results (default: 10)')\n parser.add_argument('--verbose', '-v', action='store_true', help='Show scoring details for search results')\n parser.add_argument('--clear-cache', action='store_true', help='Clear cache before operation (forces rebuild)')\n\n subparsers = parser.add_subparsers(dest='command', help='Command to execute')\n\n # Resolve command\n resolve_parser = subparsers.add_parser('resolve', help='Resolve doc_id to file path')\n resolve_parser.add_argument('doc_id', help='Document ID to resolve')\n resolve_parser.add_argument('--extract-path', help='Optional extract path')\n\n # Content command\n content_parser = subparsers.add_parser('content', help='Get document content (full or partial section)')\n content_parser.add_argument('doc_id', help='Document ID')\n content_parser.add_argument('--section', help='Optional section heading to extract')\n\n # Search command\n search_parser = subparsers.add_parser('search', help='Search by keywords')\n search_parser.add_argument('keywords', nargs='+', help='Keywords to search for')\n search_parser.add_argument('--category', help='Filter by category')\n search_parser.add_argument('--tags', nargs='+', help='Filter by tags')\n\n # Query command\n query_parser = subparsers.add_parser('query', help='Natural language search')\n query_parser.add_argument('query', help='Natural language query')\n\n # Category command\n category_parser = subparsers.add_parser('category', help='List documents by category')\n category_parser.add_argument('category', help='Category name')\n\n # Tag command\n tag_parser = subparsers.add_parser('tag', help='List documents by tag')\n tag_parser.add_argument('tag', help='Tag name')\n\n # Related command\n related_parser = subparsers.add_parser('related', help='Find related documents')\n related_parser.add_argument('doc_id', help='Document ID to find related docs for')\n\n args = parser.parse_args()\n\n if not args.command:\n parser.print_help()\n sys.exit(EXIT_BAD_ARGS)\n\n # Log script start\n logger.start({\n 'command': args.command,\n 'base_dir': args.base_dir,\n 'json': args.json\n })\n\n exit_code = EXIT_SUCCESS\n result_count = 0\n try:\n # Resolve base directory\n base_dir = resolve_base_dir(args.base_dir)\n\n # Clear cache if requested\n if getattr(args, 'clear_cache', False):\n try:\n from utils.cache_manager import CacheManager\n cm = CacheManager(base_dir)\n cm.clear_inverted_index()\n print('Cache cleared. Rebuilding index...\\n')\n except ImportError:\n print('Warning: CacheManager not available, skipping cache clear')\n\n # Initialize resolver\n resolver = DocResolver(base_dir)\n\n # Execute command and capture result count\n if args.command == 'resolve':\n result_count = cmd_resolve(resolver, args.doc_id, getattr(args, 'extract_path', None), args.json)\n elif args.command == 'content':\n result_count = cmd_content(resolver, args.doc_id, getattr(args, 'section', None), args.json)\n elif args.command == 'search':\n result_count = cmd_search(resolver, args.keywords, getattr(args, 'category', None),\n getattr(args, 'tags', None), args.limit, args.json, args.verbose)\n elif args.command == 'query':\n result_count = cmd_query(resolver, args.query, args.limit, args.json, args.verbose)\n elif args.command == 'category':\n result_count = cmd_category(resolver, args.category, args.json)\n elif args.command == 'tag':\n result_count = cmd_tag(resolver, args.tag, args.json)\n elif args.command == 'related':\n result_count = cmd_related(resolver, args.doc_id, args.limit, args.json)\n else:\n parser.print_help()\n exit_code = 1\n\n logger.end(exit_code=exit_code, summary={'results_found': result_count})\n\n except SystemExit:\n raise\n except Exception as e:\n logger.log_error(\"Fatal error in find_docs\", error=e)\n exit_code = 1\n logger.end(exit_code=exit_code)\n sys.exit(exit_code)\n\n\nif __name__ == '__main__':\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":16924,"content_sha256":"1fa2ca718dc1083101e1cf2f2ae354ef91a3d6ee9b0856c4eeba0b7ac36c3acf"},{"filename":"scripts/core/llms_parser.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nllms_parser.py - Parse llms.txt and llms-full.txt formats.\n\nAdapted for OpenAI Codex CLI Documentation from google-ecosystem gemini-cli-docs skill.\n\nThis module provides parsers for the llms.txt ecosystem:\n- llms.txt: Discovery index with markdown links [Title](URL): Description\n- llms-full.txt: Full rendered documentation with # Title / Source: URL headers\n\nUsage:\n from core.llms_parser import LlmsParser, LlmsFullParser, LlmsEntry, LlmsFullPage\n\n # Parse llms.txt for URL discovery\n parser = LlmsParser()\n for entry in parser.parse(content):\n print(f\"{entry.title}: {entry.url}\")\n\n # Stream parse llms-full.txt for content (memory-efficient)\n full_parser = LlmsFullParser()\n for page in full_parser.parse_stream(content):\n print(f\"{page.title}: {len(page.content)} chars\")\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n\nimport re\nfrom dataclasses import dataclass\nfrom typing import Generator\n\n\n@dataclass\nclass LlmsEntry:\n \"\"\"Entry from llms.txt discovery file.\"\"\"\n title: str\n url: str\n description: str | None = None\n section: str | None = None\n\n\n@dataclass\nclass LlmsFullPage:\n \"\"\"Single page from llms-full.txt content file.\"\"\"\n title: str\n source_url: str\n content: str\n\n\nclass LlmsParser:\n \"\"\"\n Parser for llms.txt discovery index format.\n\n llms.txt format (standard):\n # Site Title\n ## Section\n - [Page Title](https://example.com/page.md): Optional description\n - [Another Page](https://example.com/another.md)\n\n Also supports embedded markdown links:\n # [Page Title](http://example.com/docs/page.md)\n Content with [inline links](/docs/other.md)...\n\n This format is similar to docs-map but with optional descriptions.\n \"\"\"\n\n # Pattern: - [Title](URL): Optional description (standard format)\n # Also handles - [Title](URL) without description\n ENTRY_PATTERN = re.compile(\n r'^-\\s*\\[([^\\]]+)\\]\$(https?://[^\$]+)\\)(?::\\s*(.*))?

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

\n )\n\n # Pattern: # [Title](URL) - header with embedded link\n HEADER_LINK_PATTERN = re.compile(\n r'^#+\\s*\\[([^\\]]+)\\]\$(https?://[^\$]+)\\)\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

\n )\n\n # Pattern: Any markdown link [text](url) - for extracting all links\n # Matches both absolute URLs and relative paths\n INLINE_LINK_PATTERN = re.compile(\n r'\\[([^\\]]+)\\]\$((?:https?://[^\$]+)|(?:/[^\\)]+\\.md))\\)'\n )\n\n # Section header: ## Section Name\n SECTION_PATTERN = re.compile(r'^##\\s+(.+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n\n def __init__(self, base_url: str | None = None):\n \"\"\"\n Initialize parser.\n\n Args:\n base_url: Base URL for resolving relative paths (e.g., \"https://developers.openai.com\")\n \"\"\"\n self.base_url = base_url.rstrip('/') if base_url else None\n\n def _resolve_url(self, url: str) -> str:\n \"\"\"Resolve relative URLs to absolute using base_url.\"\"\"\n if url.startswith('http://') or url.startswith('https://'):\n return url\n if self.base_url and url.startswith('/'):\n return f\"{self.base_url}{url}\"\n return url\n\n def parse(self, content: str) -> Generator[LlmsEntry, None, None]:\n \"\"\"\n Parse llms.txt content, yielding entries.\n\n Args:\n content: Full text content of llms.txt file\n\n Yields:\n LlmsEntry objects for each documentation link found\n \"\"\"\n current_section = None\n seen_urls: set[str] = set()\n\n for line in content.splitlines():\n line_stripped = line.strip()\n\n if not line_stripped:\n continue\n\n # Check for section header (without link)\n section_match = self.SECTION_PATTERN.match(line_stripped)\n if section_match and '[' not in line_stripped:\n current_section = section_match.group(1).strip()\n continue\n\n # Check for standard entry format: - [Title](URL)\n entry_match = self.ENTRY_PATTERN.match(line_stripped)\n if entry_match:\n title = entry_match.group(1).strip()\n url = self._resolve_url(entry_match.group(2).strip())\n description = entry_match.group(3)\n if description:\n description = description.strip()\n\n if url not in seen_urls:\n seen_urls.add(url)\n yield LlmsEntry(\n title=title,\n url=url,\n description=description if description else None,\n section=current_section\n )\n continue\n\n # Check for header with embedded link: # [Title](URL)\n header_match = self.HEADER_LINK_PATTERN.match(line_stripped)\n if header_match:\n title = header_match.group(1).strip()\n url = self._resolve_url(header_match.group(2).strip())\n\n if url not in seen_urls:\n seen_urls.add(url)\n yield LlmsEntry(\n title=title,\n url=url,\n description=None,\n section=current_section\n )\n continue\n\n # Extract inline markdown links from content\n for match in self.INLINE_LINK_PATTERN.finditer(line):\n title = match.group(1).strip()\n url = self._resolve_url(match.group(2).strip())\n\n # Only include .md URLs (documentation links)\n if url.endswith('.md') and url not in seen_urls:\n seen_urls.add(url)\n yield LlmsEntry(\n title=title,\n url=url,\n description=None,\n section=current_section\n )\n\n def parse_to_list(self, content: str) -> list[LlmsEntry]:\n \"\"\"Parse llms.txt and return all entries as a list.\"\"\"\n return list(self.parse(content))\n\n def extract_urls(self, content: str) -> list[str]:\n \"\"\"Extract just the URLs from llms.txt content.\"\"\"\n return [entry.url for entry in self.parse(content)]\n\n def extract_urls_by_section(self, content: str) -> dict[str | None, list[str]]:\n \"\"\"Extract URLs grouped by section.\"\"\"\n sections: dict[str | None, list[str]] = {}\n for entry in self.parse(content):\n if entry.section not in sections:\n sections[entry.section] = []\n sections[entry.section].append(entry.url)\n return sections\n\n\nclass LlmsFullParser:\n \"\"\"\n Stream parser for llms-full.txt content format.\n\n llms-full.txt format (each page separated by title/source headers):\n # Page Title\n Source: https://example.com/page.md\n\n [Full markdown content of the page...]\n\n # Next Page Title\n Source: https://example.com/next.md\n\n [Next page content...]\n\n This parser is memory-efficient for large files (6M+ tokens).\n \"\"\"\n\n # Title pattern: # Page Title (at start of line)\n TITLE_PATTERN = re.compile(r'^#\\s+(.+)

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

)\n\n # Source pattern: Source: URL\n SOURCE_PATTERN = re.compile(r'^Source:\\s*(https?://\\S+)\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, re.IGNORECASE)\n\n def parse_stream(self, content: str) -> Generator[LlmsFullPage, None, None]:\n \"\"\"\n Stream parse llms-full.txt, yielding pages one at a time.\n\n Memory-efficient for large files - only holds one page in memory at a time.\n\n Args:\n content: Full text content of llms-full.txt file\n\n Yields:\n LlmsFullPage objects for each documentation page found\n \"\"\"\n current_title: str | None = None\n current_source: str | None = None\n content_lines: list[str] = []\n in_content = False\n\n for line in content.splitlines():\n # Check for new page title (# Title at start of line)\n title_match = self.TITLE_PATTERN.match(line)\n if title_match:\n # Yield previous page if exists and valid\n if current_title and current_source:\n yield LlmsFullPage(\n title=current_title,\n source_url=current_source,\n content='\\n'.join(content_lines).strip()\n )\n\n # Start new page\n current_title = title_match.group(1).strip()\n current_source = None\n content_lines = []\n in_content = False\n continue\n\n # Check for source URL (must follow title)\n source_match = self.SOURCE_PATTERN.match(line)\n if source_match and current_title and not current_source:\n current_source = source_match.group(1).strip()\n in_content = True\n continue\n\n # Accumulate content (only after we have title and source)\n if in_content and current_source:\n content_lines.append(line)\n\n # Yield final page\n if current_title and current_source:\n yield LlmsFullPage(\n title=current_title,\n source_url=current_source,\n content='\\n'.join(content_lines).strip()\n )\n\n def parse_to_list(self, content: str) -> list[LlmsFullPage]:\n \"\"\"Parse llms-full.txt and return all pages as a list.\"\"\"\n return list(self.parse_stream(content))\n\n def count_pages(self, content: str) -> int:\n \"\"\"Count pages without storing all content.\"\"\"\n count = 0\n for _ in self.parse_stream(content):\n count += 1\n return count\n\n def get_page_by_url(self, content: str, target_url: str) -> LlmsFullPage | None:\n \"\"\"Find a specific page by URL.\"\"\"\n for page in self.parse_stream(content):\n if page.source_url == target_url:\n return page\n return None\n\n\ndef parse_llms_txt(content: str, base_url: str | None = None) -> list[str]:\n \"\"\"\n Convenience function to parse llms.txt and extract URLs.\n\n Compatible with the existing parse_docs_map pattern.\n\n Args:\n content: llms.txt file content\n base_url: Base URL for resolving relative paths (e.g., \"https://developers.openai.com\")\n\n Returns:\n List of documentation URLs\n \"\"\"\n parser = LlmsParser(base_url=base_url)\n return parser.extract_urls(content)\n\n\ndef parse_llms_full_txt(content: str) -> Generator[LlmsFullPage, None, None]:\n \"\"\"\n Convenience function to stream parse llms-full.txt.\n\n Args:\n content: llms-full.txt file content\n\n Yields:\n LlmsFullPage objects\n \"\"\"\n parser = LlmsFullParser()\n yield from parser.parse_stream(content)\n\n\ndef url_to_local_path(source_url: str, base_dir: str | Path) -> Path:\n \"\"\"\n Convert a documentation URL to a local file path.\n\n This helper extracts the path from a URL and combines it with a base directory\n to produce a local file path for storing the documentation.\n\n Args:\n source_url: Full URL like \"https://developers.openai.com/codex/overview.md\"\n base_dir: Base output directory (can be string or Path)\n\n Returns:\n Local path like \"\u003cbase_dir>/codex/overview.md\"\n\n Example:\n >>> url_to_local_path(\"https://developers.openai.com/codex/overview.md\", Path(\"/canonical\"))\n PosixPath('/canonical/codex/overview.md')\n \"\"\"\n from urllib.parse import urlparse\n\n if isinstance(base_dir, str):\n base_dir = Path(base_dir)\n\n parsed = urlparse(source_url)\n path = parsed.path\n\n # Remove leading slash\n if path.startswith('/'):\n path = path[1:]\n\n # Ensure .md extension\n if not path.endswith('.md'):\n path = path + '.md'\n\n return base_dir / path\n\n\nif __name__ == '__main__':\n \"\"\"Self-test for llms_parser module.\"\"\"\n print(\"llms_parser Self-Test (OpenAI Codex CLI Docs)\")\n print(\"=\" * 60)\n\n # Test LlmsParser with standard format\n print(\"\\n1. Testing LlmsParser with standard llms.txt format:\")\n sample_llms_txt = \"\"\"# OpenAI Codex CLI Documentation\n\n## Getting Started\n- [Introduction](https://developers.openai.com/codex/intro.md): Welcome to Codex CLI\n- [Quickstart](https://developers.openai.com/codex/quickstart.md): Get started quickly\n\n## Configuration\n- [Configuration Guide](https://developers.openai.com/codex/configuration.md): Configure Codex\n- [Environment Variables](https://developers.openai.com/codex/env-vars.md)\n\"\"\"\n\n parser = LlmsParser()\n entries = parser.parse_to_list(sample_llms_txt)\n print(f\" Found {len(entries)} entries\")\n for entry in entries:\n print(f\" - [{entry.section}] {entry.title}: {entry.url}\")\n if entry.description:\n print(f\" Description: {entry.description}\")\n\n # Test URL extraction\n urls = parser.extract_urls(sample_llms_txt)\n print(f\"\\n Extracted {len(urls)} URLs\")\n\n # Test LlmsParser with embedded link format\n print(\"\\n2. Testing LlmsParser with embedded link format:\")\n sample_embedded_llms_txt = \"\"\"# OpenAI Codex CLI Documentation\n\n# [Codex CLI Architecture Overview](https://developers.openai.com/codex/architecture.md)\n\nThis document provides a high-level overview.\n\n## Core components\n\nThe Codex CLI is primarily composed of:\n1. **CLI package:**\n - [Command processing](/codex/commands.md)\n - [Configuration settings](/codex/configuration.md)\n\n# [Welcome to Codex CLI documentation](https://developers.openai.com/codex/index.md)\n\nThis documentation provides a comprehensive guide.\n\"\"\"\n\n parser2 = LlmsParser(base_url=\"https://developers.openai.com\")\n entries2 = parser2.parse_to_list(sample_embedded_llms_txt)\n print(f\" Found {len(entries2)} entries\")\n for entry in entries2:\n print(f\" - {entry.title}: {entry.url}\")\n\n # Test LlmsFullParser\n print(\"\\n3. Testing LlmsFullParser with sample llms-full.txt content:\")\n sample_llms_full_txt = \"\"\"# Introduction\nSource: https://developers.openai.com/codex/intro.md\n\nWelcome to Codex CLI documentation.\n\nThis is the introduction page with full content.\n\n# Quickstart\nSource: https://developers.openai.com/codex/quickstart.md\n\nGet started with Codex CLI in minutes.\n\n## Prerequisites\n- Node.js 18+\n- OpenAI API key\n\n## Installation\nnpm install -g @openai/codex\n\"\"\"\n\n full_parser = LlmsFullParser()\n pages = full_parser.parse_to_list(sample_llms_full_txt)\n print(f\" Found {len(pages)} pages\")\n for page in pages:\n print(f\" - {page.title}\")\n print(f\" Source: {page.source_url}\")\n print(f\" Content length: {len(page.content)} chars\")\n\n print(\"\\n\" + \"=\" * 60)\n print(\"Self-test complete!\")\n","content_type":"text/x-python; charset=utf-8","language":"python","size":14839,"content_sha256":"3294c220b25e7614ff90153f983360dc85fcfa5d512c2966ebf31b256ce41973"},{"filename":"scripts/core/scrape_docs.py","content":"#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nscrape_docs.py - Fetch documentation from llms.txt and URLs\n\nAdapted for OpenAI Codex CLI Documentation from google-ecosystem gemini-cli-docs skill.\n\nAutomates documentation scraping from:\n- llms.txt files (OpenAI Codex CLI documentation index)\n- Individual URLs\n\nUpdates index.yaml with metadata tracking (source URL, hash, fetch date).\n\nUsage:\n # Scrape from llms.txt\n python scrape_docs.py --llms-txt https://developers.openai.com/codex/llms.txt\n\n # Scrape specific URL\n python scrape_docs.py --url https://developers.openai.com/codex/overview.md \\\\\n --output developers-openai-com/codex/overview.md\n\nDependencies:\n pip install requests beautifulsoup4 markdownify pyyaml\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nsys.path.insert(0, str(Path(__file__).resolve().parents[1]))\n\nimport argparse\nimport hashlib\nimport json\nimport os\nimport re\nimport subprocess\nimport threading\nimport time\nimport xml.etree.ElementTree as ET\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom datetime import datetime, timedelta, timezone\nfrom urllib.parse import urlparse\n\nfrom utils.script_utils import configure_utf8_output, format_duration, HTTP_STATUS_RATE_LIMITED\nfrom utils.path_config import get_base_dir, get_index_path\nfrom utils.config_helpers import (\n get_scraper_user_agent,\n get_index_lock_retry_delay,\n get_index_lock_retry_backoff,\n get_scraping_rate_limit,\n get_scraping_header_rate_limit,\n get_scraping_max_workers,\n get_scraping_progress_lock_timeout,\n get_scraping_index_lock_timeout,\n get_http_timeout,\n get_http_max_retries,\n get_http_initial_retry_delay,\n get_http_markdown_request_timeout,\n get_validation_timeout,\n get_scraping_progress_interval,\n get_scraping_progress_url_interval,\n get_url_exclusion_patterns\n)\nfrom utils.http_utils import fetch_with_retry\nconfigure_utf8_output()\n\n# Lock retry delays (loaded from config via config_helpers)\nLOCK_RETRY_DELAY = get_index_lock_retry_delay() # Delay between lock acquisition attempts\nLOCK_RETRY_BACKOFF = get_index_lock_retry_backoff() # Delay after failed lock acquisition\n\n# Ensure unbuffered output for real-time streaming\nif sys.stdout.isatty():\n # If running in terminal, use line buffering\n sys.stdout.reconfigure(line_buffering=True)\nelse:\n # If piped (e.g., from subprocess), force unbuffered\n sys.stdout.reconfigure(line_buffering=True)\n\n# Thread-safe print helper for parallel processing (threading imported above)\n_print_lock = threading.Lock()\n\ndef safe_print(*args, **kwargs):\n \"\"\"Thread-safe print that flushes immediately for real-time output\"\"\"\n with _print_lock:\n print(*args, **kwargs, flush=True)\n\nfrom utils.logging_utils import get_or_setup_logger\n\n# Get source name from environment (set by scrape_all_sources.py for parallel worker identification)\n_source_name = os.environ.get('CODEX_DOCS_SOURCE_NAME', '')\n_log_prefix = f\"[{_source_name}] \" if _source_name else \"\"\n\nlogger = get_or_setup_logger(__file__, log_category=\"scrape\")\n\nfrom utils.script_utils import ensure_yaml_installed\n\ntry:\n import requests\n from bs4 import BeautifulSoup\n from markdownify import markdownify as md\nexcept ImportError as e:\n print(f\"Missing dependency: {e}\")\n print(\"Install with: pip install requests beautifulsoup4 markdownify\")\n sys.exit(1)\n\nyaml = ensure_yaml_installed()\n\n\nclass RateLimiter:\n \"\"\"Thread-safe rate limiter for controlling request frequency\"\"\"\n\n def __init__(self, delay: float):\n \"\"\"\n Initialize rate limiter\n\n Args:\n delay: Minimum delay between requests in seconds\n \"\"\"\n self.delay = delay\n self.lock = threading.Lock()\n self.last_request_time = 0.0\n\n def wait(self):\n \"\"\"Wait if necessary to maintain rate limit\"\"\"\n with self.lock:\n current_time = time.time()\n time_since_last = current_time - self.last_request_time\n if time_since_last \u003c self.delay:\n sleep_time = self.delay - time_since_last\n time.sleep(sleep_time)\n self.last_request_time = time.time()\n\n# Import index_manager for large file support\ntry:\n from management.index_manager import IndexManager\nexcept ImportError:\n # Fallback if index_manager not available\n IndexManager = None\n\n# Import metadata extractor\ntry:\n from management.extract_metadata import MetadataExtractor\nexcept ImportError:\n MetadataExtractor = None\n\n\nclass DocScraper:\n \"\"\"Documentation scraper with llms.txt and URL support\"\"\"\n\n def __init__(self, base_output_dir: Path | None = None, rate_limit: float | None = None,\n header_rate_limit: float | None = None, trust_existing: bool = False,\n skip_age_days: int = 0, max_workers: int | None = None, try_markdown: bool = True):\n \"\"\"\n Initialize scraper\n\n Args:\n base_output_dir: Base directory for canonical storage. If None, uses config default.\n rate_limit: Delay between requests in seconds. If None, uses config default.\n header_rate_limit: Delay between HEAD requests in seconds. If None, uses config default.\n trust_existing: If True, skip hash check when HTTP headers unavailable (default: False)\n skip_age_days: Skip files fetched within this many days if hash matches (default: 0 = today only)\n max_workers: Maximum parallel workers for URL processing. If None, uses config default.\n try_markdown: If True, try fetching .md URLs before HTML conversion (default: True)\n \"\"\"\n # Use config defaults if not provided\n self.base_output_dir = base_output_dir if base_output_dir else get_base_dir()\n self.rate_limit = rate_limit if rate_limit is not None else get_scraping_rate_limit()\n self.header_rate_limit = header_rate_limit if header_rate_limit is not None else get_scraping_header_rate_limit()\n self.trust_existing = trust_existing\n self.skip_age_days = skip_age_days\n self.max_workers = max_workers if max_workers is not None else get_scraping_max_workers()\n self.try_markdown = try_markdown\n self.index_path = get_index_path(self.base_output_dir)\n self.progress_file = self.base_output_dir / \".scrape_progress.json\"\n\n # Thread-safe rate limiters (use instance values, not parameters)\n self.rate_limiter = RateLimiter(self.rate_limit)\n self.header_rate_limiter = RateLimiter(self.header_rate_limit)\n\n # Use thread-local sessions for thread safety\n self._session_local = threading.local()\n self.session_headers = {\n 'User-Agent': get_scraper_user_agent()\n }\n\n # Initialize index manager if available\n if IndexManager:\n self.index_manager = IndexManager(base_output_dir)\n else:\n self.index_manager = None\n\n # Track 404 URLs for drift detection\n self.url_404s: set[str] = set()\n\n # Track skip reasons for observability (thread-safe counters)\n self._skip_lock = threading.Lock()\n self.skip_reasons: dict[str, int] = {\n 'http_headers_unchanged': 0,\n 'trust_existing_no_headers': 0,\n 'content_hash_unchanged': 0,\n 'fetched_within_age': 0,\n 'fetched_today': 0,\n 'resume_already_scraped': 0,\n }\n\n @property\n def session(self):\n \"\"\"Get thread-local session\"\"\"\n if not hasattr(self._session_local, 'session'):\n self._session_local.session = requests.Session()\n self._session_local.session.headers.update(self.session_headers)\n return self._session_local.session\n\n def _track_skip(self, reason: str) -> None:\n \"\"\"Thread-safe skip reason tracking for observability.\"\"\"\n with self._skip_lock:\n if reason in self.skip_reasons:\n self.skip_reasons[reason] += 1\n else:\n self.skip_reasons[reason] = 1\n\n def get_skip_summary(self) -> str:\n \"\"\"Get formatted summary of skip reasons for logging.\"\"\"\n with self._skip_lock:\n active_reasons = {k: v for k, v in self.skip_reasons.items() if v > 0}\n if not active_reasons:\n return \"No skips\"\n parts = [f\"{k}={v}\" for k, v in active_reasons.items()]\n return f\"SKIP REASONS: {', '.join(parts)}\"\n\n def filter_excluded_urls(self, urls: list[str]) -> list[str]:\n \"\"\"Filter out URLs matching exclusion patterns from config.\"\"\"\n exclusion_patterns = get_url_exclusion_patterns()\n if not exclusion_patterns:\n return urls\n\n compiled_patterns = [re.compile(pattern) for pattern in exclusion_patterns]\n\n filtered_urls = []\n excluded_count = 0\n for url in urls:\n excluded = False\n for pattern in compiled_patterns:\n if pattern.search(url):\n excluded = True\n excluded_count += 1\n break\n if not excluded:\n filtered_urls.append(url)\n\n if excluded_count > 0:\n print(f\" Excluded {excluded_count} URLs matching exclusion patterns\")\n\n return filtered_urls\n\n def load_progress(self) -> set[str]:\n \"\"\"Load already-scraped URLs from progress file (parallel-safe with locking)\"\"\"\n if not self.progress_file.exists():\n return set()\n\n lock_file = self.progress_file.parent / '.progress.lock'\n start_time = time.time()\n timeout = get_scraping_progress_lock_timeout()\n\n # Acquire lock\n while time.time() - start_time \u003c timeout:\n try:\n fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)\n os.close(fd)\n break\n except OSError:\n time.sleep(LOCK_RETRY_DELAY)\n continue\n else:\n print(f\" Warning: Could not acquire progress lock, retrying...\")\n return set()\n\n try:\n try:\n with open(self.progress_file, 'r', encoding='utf-8') as f:\n return set(json.load(f))\n except Exception as e:\n print(f\" Warning: Error loading progress: {e}\")\n return set()\n finally:\n try:\n if lock_file.exists():\n lock_file.unlink()\n except Exception:\n pass\n\n def save_progress(self, url: str):\n \"\"\"Save successfully scraped URL (parallel-safe with locking)\"\"\"\n lock_file = self.progress_file.parent / '.progress.lock'\n start_time = time.time()\n timeout = get_scraping_progress_lock_timeout()\n\n while time.time() - start_time \u003c timeout:\n try:\n fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)\n os.close(fd)\n break\n except OSError:\n time.sleep(LOCK_RETRY_DELAY)\n continue\n else:\n print(f\" Warning: Could not acquire progress lock, skipping save...\")\n return\n\n try:\n progress = self.load_progress()\n progress.add(url)\n try:\n with open(self.progress_file, 'w', encoding='utf-8') as f:\n json.dump(list(progress), f, indent=2)\n except Exception as e:\n print(f\" Warning: Error saving progress: {e}\")\n finally:\n try:\n if lock_file.exists():\n lock_file.unlink()\n except Exception:\n pass\n\n def clear_progress(self):\n \"\"\"Clear progress file\"\"\"\n if self.progress_file.exists():\n try:\n self.progress_file.unlink()\n except Exception:\n pass\n\n def fetch_url(self, url: str, max_retries: int | None = None, base_delay: float | None = None) -> tuple[str | None, str | None]:\n \"\"\"Fetch content from URL with retry logic and exponential backoff\"\"\"\n if max_retries is None:\n max_retries = get_http_max_retries()\n if base_delay is None:\n base_delay = get_http_initial_retry_delay()\n\n for attempt in range(max_retries):\n try:\n if attempt == 0:\n print(f\" Fetching: {url}\")\n else:\n print(f\" Fetching: {url} (attempt {attempt + 1}/{max_retries})\")\n\n self.rate_limiter.wait()\n http_timeout = get_http_timeout()\n response = self.session.get(url, timeout=http_timeout)\n response.raise_for_status()\n return (response.text, response.url)\n\n except requests.HTTPError as e:\n status_code = e.response.status_code if e.response else None\n\n if status_code == 404:\n print(f\" 404 Not Found: {url}\")\n self.url_404s.add(url)\n return (None, None)\n\n elif status_code == HTTP_STATUS_RATE_LIMITED:\n if attempt \u003c max_retries - 1:\n retry_after = int(e.response.headers.get('Retry-After', base_delay * (2 ** attempt)))\n wait_time = max(retry_after, base_delay * (2 ** attempt))\n print(f\" Rate limited (429), retrying in {wait_time}s...\")\n time.sleep(wait_time)\n continue\n else:\n print(f\" Rate limit exceeded after {max_retries} attempts: {url}\")\n return (None, None)\n\n elif status_code and status_code >= 500:\n if attempt \u003c max_retries - 1:\n wait_time = base_delay * (2 ** attempt)\n print(f\" Server error {status_code} (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s...\")\n time.sleep(wait_time)\n continue\n else:\n print(f\" Server error {status_code} after {max_retries} attempts: {url}\")\n return (None, None)\n\n else:\n print(f\" HTTP {status_code}: {url}\")\n return (None, None)\n\n except requests.ConnectionError:\n if attempt \u003c max_retries - 1:\n wait_time = base_delay * (2 ** attempt)\n print(f\" Connection error (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s...\")\n time.sleep(wait_time)\n continue\n else:\n print(f\" Connection error after {max_retries} attempts: {url}\")\n return (None, None)\n\n except requests.Timeout:\n if attempt \u003c max_retries - 1:\n wait_time = base_delay * (2 ** attempt)\n print(f\" Timeout (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s...\")\n time.sleep(wait_time)\n continue\n else:\n print(f\" Timeout after {max_retries} attempts: {url}\")\n return (None, None)\n\n except requests.RequestException:\n if attempt \u003c max_retries - 1:\n wait_time = base_delay * (2 ** attempt)\n print(f\" Request error (attempt {attempt + 1}/{max_retries}), retrying in {wait_time}s...\")\n time.sleep(wait_time)\n continue\n else:\n print(f\" Request error after {max_retries} attempts: {url}\")\n return (None, None)\n\n return (None, None)\n\n def try_fetch_markdown(self, url: str) -> tuple[str | None, str | None, str | None]:\n \"\"\"Try to fetch clean markdown from URL with retry logic\"\"\"\n if not self.try_markdown:\n return (None, None, None)\n\n try:\n if url.endswith('.md'):\n markdown_url = url\n final_url = url.removesuffix('.md')\n print(f\" Trying markdown URL (direct): {markdown_url}\")\n else:\n head_timeout = get_http_timeout()\n self.rate_limiter.wait()\n head_response = self.session.head(url, timeout=head_timeout, allow_redirects=True)\n final_url = head_response.url\n\n if final_url != url:\n print(f\" Redirected: {url} -> {final_url}\")\n\n if final_url.endswith('/'):\n final_url = final_url[:-1]\n\n if final_url.endswith('.md'):\n markdown_url = final_url\n else:\n markdown_url = f\"{final_url}.md\"\n\n print(f\" Trying markdown URL: {markdown_url}\")\n\n markdown_timeout = get_http_markdown_request_timeout()\n max_retries = get_http_max_retries()\n initial_delay = get_http_initial_retry_delay()\n\n response = fetch_with_retry(\n markdown_url,\n max_retries=max_retries,\n initial_delay=initial_delay,\n timeout=markdown_timeout,\n session=self.session\n )\n\n time.sleep(self.rate_limiter.delay)\n\n content = response.text\n\n content_stripped = content.strip()\n if content_stripped.startswith('#') or content_stripped.startswith('---'):\n print(f\" Successfully fetched clean markdown from {markdown_url}\")\n return (content, \"markdown\", final_url)\n else:\n print(f\" URL returned content but doesn't appear to be markdown\")\n return (None, None, None)\n\n except requests.HTTPError as e:\n status_code = e.response.status_code if e.response else None\n failed_url = e.request.url if e.request else \"unknown\"\n\n if status_code == 404:\n if 'markdown_url' in locals() and failed_url == markdown_url:\n self.url_404s.add(markdown_url)\n print(f\" Markdown URL 404: {markdown_url}\")\n else:\n print(f\" Base URL 404: {url}\")\n else:\n status_str = str(status_code) if status_code else 'HTTP error'\n print(f\" Markdown URL not available ({status_str}), will try HTML conversion\")\n return (None, None, None)\n except (requests.ConnectionError, requests.Timeout, requests.RequestException) as e:\n error_type = type(e).__name__\n print(f\" Markdown URL not available ({error_type}), will try HTML conversion\")\n return (None, None, None)\n\n def parse_llms_txt(self, llms_txt_url: str) -> list[str]:\n \"\"\"Parse llms.txt and extract documentation URLs\"\"\"\n print(f\"Parsing llms.txt: {llms_txt_url}\")\n content, _ = self.fetch_url(llms_txt_url)\n if not content:\n return []\n\n parsed = urlparse(llms_txt_url)\n base_url = f\"{parsed.scheme}://{parsed.netloc}\"\n\n from llms_parser import LlmsParser\n parser = LlmsParser(base_url=base_url)\n urls = parser.extract_urls(content)\n\n print(f\" Found {len(urls)} documentation URLs\")\n\n urls = self.filter_excluded_urls(urls)\n\n return urls\n\n def html_to_markdown(self, html_content: str, source_url: str | None = None) -> str:\n \"\"\"Convert HTML content to markdown\"\"\"\n soup = BeautifulSoup(html_content, 'html.parser')\n\n # Tier 1: Try to extract main content container\n main_content = None\n for selector in ['main', '[role=\"main\"]', 'article']:\n main_content = soup.select_one(selector)\n if main_content:\n break\n\n if main_content:\n # Use only the main content area (excludes all nav/sidebar/footer)\n target = main_content\n else:\n # Tier 2: Remove chrome elements from full page\n target = soup\n\n # Remove known chrome elements from target\n for element in target(['script', 'style', 'nav', 'header', 'footer']):\n element.decompose()\n\n # Remove sidebar/navigation elements by CSS selector\n for selector in [\n 'aside',\n '[role=\"navigation\"]',\n '[class*=\"sidebar\"]',\n '[class*=\"nav-\"]',\n '[class*=\"site-nav\"]',\n '[class*=\"table-of-contents\"]',\n ]:\n for element in target.select(selector):\n element.decompose()\n\n markdown = md(str(target), heading_style=\"ATX\")\n\n markdown = re.sub(r'\\n\\n\\n+', '\\n\\n', markdown)\n\n return markdown.strip()\n\n def calculate_hash(self, content: str) -> str:\n \"\"\"Calculate SHA-256 hash of content\"\"\"\n hash_obj = hashlib.sha256(content.encode('utf-8'))\n return f\"sha256:{hash_obj.hexdigest()}\"\n\n def normalize_etag(self, etag: str | None) -> str | None:\n \"\"\"Normalize ETag for comparison\"\"\"\n if etag is None:\n return None\n\n if etag == '':\n return ''\n\n if etag.startswith('W/'):\n etag = etag[2:]\n\n etag = etag.strip('\"').strip(\"'\")\n\n return etag\n\n def should_skip_url(self, url: str, output_path: Path, use_http_headers: bool = True,\n verbose: bool = False) -> bool:\n \"\"\"Check if URL should be skipped (already exists with matching hash and source URL)\"\"\"\n if not output_path.exists():\n return False\n\n try:\n content = output_path.read_text(encoding='utf-8')\n if not content.startswith('---'):\n return False\n\n frontmatter_end = content.find('---', 3)\n if frontmatter_end == -1:\n return False\n\n frontmatter_text = content[3:frontmatter_end].strip()\n frontmatter = yaml.safe_load(frontmatter_text)\n\n if frontmatter.get('source_url') != url:\n return False\n\n existing_hash = frontmatter.get('content_hash')\n if existing_hash:\n if self.trust_existing:\n if verbose:\n print(f\" Skipping (trust existing): {url}\")\n self._track_skip('trust_existing_no_headers')\n return True\n\n last_fetched = frontmatter.get('last_fetched')\n if last_fetched:\n try:\n fetch_date = datetime.fromisoformat(last_fetched).date()\n today = datetime.now(timezone.utc).date()\n days_ago = (today - fetch_date).days\n\n if days_ago \u003c= self.skip_age_days:\n if existing_hash and self.trust_existing:\n if verbose:\n print(f\" Skipping (fetched {days_ago} days ago, trust existing): {url}\")\n self._track_skip('fetched_within_age')\n return True\n elif days_ago == 0:\n if verbose:\n print(f\" Skipping (fetched today): {url}\")\n self._track_skip('fetched_today')\n return True\n except (ValueError, TypeError):\n pass\n\n return False\n except Exception as e:\n if verbose:\n print(f\" Warning: Error checking skip status: {e}, will re-scrape\")\n return False\n\n def add_frontmatter(self, content: str, url: str, source_type: str,\n sitemap_url: str | None = None, fetch_method: str | None = None) -> str:\n \"\"\"Add YAML frontmatter to content\"\"\"\n content_hash = self.calculate_hash(content)\n\n frontmatter = {\n 'source_url': url,\n 'source_type': source_type,\n 'content_hash': content_hash\n }\n\n if sitemap_url:\n frontmatter['sitemap_url'] = sitemap_url\n\n if fetch_method:\n frontmatter['fetch_method'] = fetch_method\n\n yaml_frontmatter = yaml.dump(frontmatter, default_flow_style=False, sort_keys=False)\n\n return f\"---\\n{yaml_frontmatter}---\\n\\n{content}\"\n\n def scrape_url(self, url: str, output_path: Path, source_type: str,\n sitemap_url: str | None = None, skip_existing: bool = False,\n max_retries: int | None = None, max_content_age_days: int | None = None) -> bool:\n \"\"\"Scrape single URL and save with frontmatter\"\"\"\n url_start_time = time.time()\n\n if max_retries is None:\n max_retries = get_http_max_retries()\n\n if skip_existing:\n if self.should_skip_url(url, output_path, use_http_headers=True, verbose=True):\n return True\n\n markdown, fetch_method, final_url = self.try_fetch_markdown(url)\n\n if final_url:\n if final_url != url:\n url = final_url\n try:\n new_output_subdir = self.auto_detect_output_dir(url)\n new_output_dir = self.base_output_dir / new_output_subdir\n new_relative_path = self.url_to_filename(url, base_pattern=None)\n output_path = new_output_dir / new_relative_path\n output_path.parent.mkdir(parents=True, exist_ok=True)\n print(f\" Updated output path due to redirect: {output_path}\")\n except Exception as e:\n print(f\" Warning: Failed to update output path for redirect {url}: {e}\")\n\n markdown_url = f\"{url}.md\" if not url.endswith('.md') else url\n if markdown is None and (url in self.url_404s or markdown_url in self.url_404s):\n return False\n\n if markdown is None:\n html_content, final_html_url = self.fetch_url(url)\n\n if not html_content:\n print(f\" Failed to fetch HTML from {url}\")\n return False\n\n if final_html_url and final_html_url != url:\n url = final_html_url\n try:\n new_output_subdir = self.auto_detect_output_dir(url)\n new_output_dir = self.base_output_dir / new_output_subdir\n new_relative_path = self.url_to_filename(url, base_pattern=None)\n output_path = new_output_dir / new_relative_path\n output_path.parent.mkdir(parents=True, exist_ok=True)\n print(f\" Updated output path due to HTML redirect: {output_path}\")\n except Exception as e:\n print(f\" Warning: Failed to update output path for HTML redirect {url}: {e}\")\n\n markdown = self.html_to_markdown(html_content, url)\n fetch_method = \"html\"\n\n if output_path.exists():\n try:\n existing_content = output_path.read_text(encoding='utf-8')\n if existing_content.startswith('---'):\n fm_end = existing_content.find('---', 3)\n if fm_end != -1:\n existing_fm = yaml.safe_load(existing_content[3:fm_end])\n existing_hash = existing_fm.get('content_hash')\n if existing_hash:\n new_hash = self.calculate_hash(markdown)\n if existing_hash == new_hash:\n print(f\" Skipping (content unchanged, metadata-only diff): {url}\")\n self._track_skip('content_unchanged_pre_write')\n return True\n except Exception:\n pass\n\n final_content = self.add_frontmatter(markdown, url, source_type, sitemap_url, fetch_method)\n\n output_path.parent.mkdir(parents=True, exist_ok=True)\n\n if not final_content.endswith('\\n'):\n final_content += '\\n'\n\n with open(output_path, 'w', encoding='utf-8', newline='\\n') as f:\n f.write(final_content)\n safe_print(f\" Saved: {output_path}\")\n\n try:\n relative_to_base = output_path.relative_to(self.base_output_dir)\n path_normalized = str(relative_to_base).replace('\\\\', '/')\n except ValueError:\n path_normalized = str(output_path).replace('\\\\', '/')\n\n metadata = {\n 'path': path_normalized,\n 'url': url,\n 'hash': self.calculate_hash(final_content),\n 'last_fetched': datetime.now(timezone.utc).strftime('%Y-%m-%d'),\n 'source_type': source_type,\n 'sitemap_url': sitemap_url\n }\n\n if MetadataExtractor:\n try:\n extractor = MetadataExtractor(output_path, url)\n extracted = extractor.extract_all()\n metadata.update(extracted)\n except Exception:\n pass\n\n url_duration_ms = (time.time() - url_start_time) * 1000\n logger.debug(f\"URL scraped in {url_duration_ms:.0f}ms: {url}\")\n return metadata\n\n def normalize_domain(self, url: str) -> str:\n \"\"\"Extract and normalize domain name for use as folder name\"\"\"\n parsed = urlparse(url)\n domain = parsed.netloc\n if domain.startswith('www.'):\n domain = domain[4:]\n return domain.replace('.', '-')\n\n def auto_detect_output_dir(self, url: str, url_filter: str | None = None) -> str:\n \"\"\"Auto-detect output directory based on URL domain\"\"\"\n from urllib.parse import urlparse\n from utils.config_helpers import get_output_dir_mapping\n\n parsed = urlparse(url)\n domain = parsed.netloc\n\n return get_output_dir_mapping(domain)\n\n def url_to_filename(self, url: str, base_pattern: str = None) -> str:\n \"\"\"Convert URL to relative filepath preserving directory structure\"\"\"\n parsed = urlparse(url)\n path = parsed.path.strip('/')\n\n if base_pattern:\n base_pattern = base_pattern.strip('/')\n if path.startswith(base_pattern):\n path = path[len(base_pattern):].strip('/')\n\n path = re.sub(r'^[a-z]{2}(-[A-Z]{2})?/', '', path)\n\n if not path.endswith('.md'):\n path += '.md'\n\n return path\n\n def update_index(self, doc_id: str, metadata: dict) -> None:\n \"\"\"Update index.yaml with document metadata (parallel-safe with file locking)\"\"\"\n if self.index_manager:\n if not self.index_manager.update_entry(doc_id, metadata):\n print(f\" Warning: Failed to update index entry: {doc_id}\")\n else:\n lock_file = self.index_path.parent / '.index.lock'\n start_time = time.time()\n timeout = get_scraping_index_lock_timeout()\n\n while time.time() - start_time \u003c timeout:\n try:\n fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)\n os.close(fd)\n break\n except OSError:\n time.sleep(LOCK_RETRY_DELAY)\n continue\n else:\n print(f\" Warning: Could not acquire index lock\")\n return\n\n try:\n if self.index_path.exists():\n with open(self.index_path, 'r', encoding='utf-8') as f:\n index = yaml.safe_load(f) or {}\n else:\n index = {}\n\n index[doc_id] = metadata\n\n with open(self.index_path, 'w', encoding='utf-8') as f:\n yaml.dump(index, f, default_flow_style=False, sort_keys=False, allow_unicode=True)\n finally:\n try:\n if lock_file.exists():\n lock_file.unlink()\n except Exception:\n pass\n\n def scrape_from_llms_txt(self, llms_txt_url: str, output_subdir: str | None = None,\n limit: int | None = None, max_age_days: int | None = None,\n skip_existing: bool = False, max_retries: int | None = None,\n auto_validate: bool = False, expected_count: int | None = None,\n resume: bool = False, skip_urls: list[str] | None = None) -> int:\n \"\"\"Scrape multiple URLs from llms.txt index\"\"\"\n if max_retries is None:\n max_retries = get_http_max_retries()\n\n urls = self.parse_llms_txt(llms_txt_url)\n\n if skip_urls:\n original_count = len(urls)\n skip_urls_set = set(skip_urls)\n original_urls_set = set(urls)\n urls = [u for u in urls if u not in skip_urls_set]\n skipped_known_bad = original_count - len(urls)\n if skipped_known_bad > 0:\n logger.info(f\"Skipped {skipped_known_bad} known-bad URL(s) from expected_errors config\")\n for skipped_url in skip_urls_set:\n if skipped_url in original_urls_set:\n logger.debug(f\" Skipped known-bad URL: {skipped_url}\")\n self._track_skip('expected_error_404')\n\n if resume:\n progress = self.load_progress()\n urls_to_scrape = [u for u in urls if u not in progress]\n skipped_count = len(urls) - len(urls_to_scrape)\n if skipped_count > 0:\n print(f\" Resuming: {skipped_count} URLs already scraped, {len(urls_to_scrape)} remaining\")\n for _ in range(skipped_count):\n self._track_skip('resume_already_scraped')\n urls = urls_to_scrape\n else:\n self.clear_progress()\n\n if limit:\n urls = urls[:limit]\n print(f\" Limiting to first {limit} URLs\")\n\n if not output_subdir:\n output_subdir = self.auto_detect_output_dir(llms_txt_url)\n print(f\" Auto-detected output directory: {output_subdir}\")\n\n output_dir = self.base_output_dir / output_subdir\n success_count = 0\n skipped_count = 0\n failed_count = 0\n failed_urls = []\n url_timings: list[float] = []\n scrape_start_time = time.time()\n last_progress_time = time.time()\n progress_interval = get_scraping_progress_interval()\n progress_url_interval = get_scraping_progress_url_interval()\n\n for i, url in enumerate(urls, 1):\n url_start = time.time()\n print(f\"\\n[{i}/{len(urls)}] Processing: {url}\")\n\n relative_path = self.url_to_filename(url, base_pattern=None)\n output_path = output_dir / relative_path\n\n was_skipped = False\n if skip_existing and self.should_skip_url(url, output_path):\n was_skipped = True\n skipped_count += 1\n\n scrape_result = self.scrape_url(url, output_path, source_type='llms-txt', sitemap_url=llms_txt_url,\n skip_existing=skip_existing, max_retries=max_retries,\n max_content_age_days=max_age_days)\n if scrape_result:\n success_count += 1\n\n if resume:\n self.save_progress(url)\n\n if not was_skipped and output_path.exists():\n if isinstance(scrape_result, dict):\n metadata = scrape_result\n path_str = metadata['path']\n doc_id_suffix = path_str.replace('.md', '').replace('/', '-')\n doc_id = doc_id_suffix\n self.update_index(doc_id, metadata)\n else:\n doc_id_suffix = relative_path.replace('.md', '').replace('/', '-')\n doc_id = f\"{output_subdir.replace('/', '-')}-{doc_id_suffix}\"\n try:\n content = output_path.read_text(encoding='utf-8')\n frontmatter = {}\n if content.startswith('---'):\n end = content.find('---', 3)\n if end > 0:\n frontmatter = yaml.safe_load(content[3:end])\n path_str = str(output_path.relative_to(self.base_output_dir)).replace('\\\\', '/')\n metadata = {\n 'source_url': url,\n 'content_hash': frontmatter.get('content_hash', ''),\n 'last_fetched': frontmatter.get('last_fetched', ''),\n 'path': path_str,\n 'doc_id': doc_id,\n 'url': url,\n 'source_type': source_type,\n 'sitemap_url': sitemap_url\n }\n self.update_index(doc_id, metadata)\n except Exception as e:\n logger.warning(f\"Could not extract metadata for index: {e}\")\n else:\n failed_count += 1\n failed_urls.append(url)\n\n url_time = time.time() - url_start\n url_timings.append(url_time)\n\n current_time = time.time()\n if current_time - last_progress_time >= progress_interval or i % progress_url_interval == 0:\n elapsed = current_time - scrape_start_time\n rate = i / elapsed if elapsed > 0 else 0\n remaining = len(urls) - i\n eta = remaining / rate if rate > 0 else 0\n print(f\" Progress: {i}/{len(urls)} ({i/len(urls)*100:.1f}%) | \"\n f\"Rate: {rate:.1f} URL/s | ETA: {format_duration(eta)}\")\n last_progress_time = current_time\n\n total_scrape_time = time.time() - scrape_start_time\n throughput = len(urls) / total_scrape_time if total_scrape_time > 0 else 0\n avg_time_ms = (sum(url_timings) / len(url_timings) * 1000) if url_timings else 0\n max_time_ms = max(url_timings) * 1000 if url_timings else 0\n min_time_ms = min(url_timings) * 1000 if url_timings else 0\n\n print(f\"\\n{'='*60}\")\n print(f\"Scraping Summary for {len(urls)} URLs:\")\n print(f\"{'='*60}\")\n new_updated = max(0, success_count - skipped_count)\n print(f\" New/Updated: {new_updated}\")\n print(f\" Skipped (hash): {skipped_count}\")\n if failed_count > 0:\n print(f\" Failed: {failed_count}\")\n print(f\" Total processed: {len(urls)}\")\n skip_summary = self.get_skip_summary()\n if skip_summary != \"No skips\":\n logger.info(skip_summary)\n print(f\"{'='*60}\")\n print(f\" Total time: {total_scrape_time:.1f}s\")\n print(f\" Throughput: {throughput:.2f} URLs/sec\")\n print(f\" Avg per URL: {avg_time_ms:.0f}ms\")\n if len(url_timings) > 1:\n print(f\" Slowest URL: {max_time_ms:.0f}ms\")\n print(f\" Fastest URL: {min_time_ms:.0f}ms\")\n print(f\"{'='*60}\")\n\n if failed_urls:\n print(f\"\\nFailed URLs ({len(failed_urls)}):\")\n for failed_url in failed_urls:\n print(f\" - {failed_url}\")\n\n return success_count\n\n\ndef main():\n \"\"\"Main entry point\"\"\"\n parser = argparse.ArgumentParser(\n description='Scrape OpenAI Codex CLI documentation from llms.txt',\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n # Scrape from llms.txt (auto-detects output directory from domain)\n python scrape_docs.py --llms-txt https://developers.openai.com/codex/llms.txt\n # Output: developers-openai-com/codex/...\n\n # Scrape single URL\n python scrape_docs.py --url https://developers.openai.com/codex/overview.md \\\\\n --output developers-openai-com/codex/overview.md\n\n # Test with limit\n python scrape_docs.py --llms-txt https://developers.openai.com/codex/llms.txt \\\\\n --limit 5\n \"\"\"\n )\n\n # Input source (mutually exclusive)\n source_group = parser.add_mutually_exclusive_group(required=True)\n source_group.add_argument('--llms-txt', help='URL to llms.txt file (markdown link index)')\n source_group.add_argument('--url', help='Single URL to scrape')\n\n # Output\n parser.add_argument('--output',\n help='Output path (auto-detected from URL if not provided; required for --url)')\n\n # Options\n parser.add_argument('--limit', type=int, help='Limit number of URLs to scrape (for testing)')\n\n # Get defaults from config\n from utils.cli_utils import add_base_dir_argument, resolve_base_dir_from_args\n default_rate_limit = get_scraping_rate_limit()\n default_header_rate_limit = get_scraping_header_rate_limit()\n default_max_workers = get_scraping_max_workers()\n\n add_base_dir_argument(parser)\n parser.add_argument('--rate-limit', type=float, default=default_rate_limit,\n help=f'Delay between requests in seconds (default: {default_rate_limit}, from config)')\n parser.add_argument('--header-rate-limit', type=float, default=default_header_rate_limit,\n help=f'Delay between HEAD requests in seconds (default: {default_header_rate_limit}, from config)')\n parser.add_argument('--max-workers', type=int, default=default_max_workers,\n help=f'Maximum parallel workers for URL processing (default: {default_max_workers}, from config)')\n parser.add_argument('--skip-existing', action='store_true',\n help='Skip URLs that already exist with matching hash and source URL (idempotent mode)')\n parser.add_argument('--trust-existing', action='store_true',\n help='Skip hash check when HTTP headers unavailable (faster, but less accurate)')\n parser.add_argument('--no-try-markdown', action='store_true',\n help='Skip trying .md URLs (go straight to HTML conversion)')\n parser.add_argument('--skip-age-days', type=int, default=0,\n help='Skip files fetched within this many days if hash matches (default: 0 = today only)')\n default_max_retries = get_http_max_retries()\n parser.add_argument('--max-retries', type=int, default=default_max_retries,\n help=f'Maximum retry attempts for transient failures (default: {default_max_retries}, from config)')\n parser.add_argument('--resume', action='store_true',\n help='Resume from last successful URL (uses .scrape_progress.json)')\n parser.add_argument('--skip-urls', type=str, nargs='*', default=[],\n help='URLs to skip (e.g., known 404s from expected_errors in sources.json)')\n\n args = parser.parse_args()\n\n # Print dev/prod mode banner for visibility\n if not _source_name:\n from utils.dev_mode import print_mode_banner\n from utils.path_config import get_base_dir\n print_mode_banner(logger)\n logger.info(f\"Canonical dir: {get_base_dir()}\")\n\n start_context = {\n 'source': 'llms_txt' if args.llms_txt else 'url',\n 'base_dir': args.base_dir,\n 'limit': args.limit,\n 'skip_existing': args.skip_existing\n }\n if _source_name:\n start_context['source_name'] = _source_name\n logger.start(start_context)\n\n exit_code = 0\n try:\n if args.url and not args.output:\n parser.error(\"--url requires --output to be specified\")\n\n base_dir = resolve_base_dir_from_args(args)\n\n base_dir.mkdir(parents=True, exist_ok=True)\n\n print(f\"Using base directory: {base_dir}\")\n print(f\" (Absolute path: {base_dir.absolute()})\")\n\n scraper = DocScraper(\n base_dir,\n rate_limit=args.rate_limit if args.rate_limit != default_rate_limit else None,\n header_rate_limit=args.header_rate_limit if args.header_rate_limit != default_header_rate_limit else None,\n max_workers=args.max_workers if args.max_workers != default_max_workers else None,\n trust_existing=args.trust_existing,\n skip_age_days=args.skip_age_days,\n try_markdown=not args.no_try_markdown\n )\n\n if args.llms_txt:\n success_count = scraper.scrape_from_llms_txt(\n args.llms_txt,\n args.output,\n limit=args.limit,\n skip_existing=args.skip_existing,\n max_retries=args.max_retries,\n resume=args.resume,\n skip_urls=args.skip_urls if args.skip_urls else None\n )\n elif args.url:\n output_path = base_dir / args.output\n result = scraper.scrape_url(args.url, output_path, source_type='manual',\n skip_existing=args.skip_existing,\n max_retries=args.max_retries)\n\n if isinstance(result, dict):\n success = True\n metadata = result\n path_str = metadata['path']\n doc_id = path_str.replace('.md', '').replace('/', '-')\n scraper.update_index(doc_id, metadata)\n elif result is True:\n success = True\n else:\n success = False\n\n success_count = 1 if success else 0\n\n print(f\"\\n{'='*60}\")\n print(f\"Scraping complete: {success_count} document(s) processed\")\n\n if args.llms_txt:\n output_subdir = args.output or scraper.auto_detect_output_dir(args.llms_txt)\n output_dir = base_dir / output_subdir\n if output_dir.exists():\n md_files = list(output_dir.glob(\"**/*.md\"))\n total_size = sum(f.stat().st_size for f in md_files)\n size_mb = total_size / 1024 / 1024\n print(f\"Files: {len(md_files)}\")\n print(f\"Size: {size_mb:.2f} MB\")\n logger.track_metric('total_files', len(md_files))\n logger.track_metric('total_size_mb', size_mb)\n\n duration_seconds = logger.performance_metrics.get('duration_seconds', 0)\n if duration_seconds > 0:\n print(f\"Total duration: {format_duration(duration_seconds)}\")\n\n logger.track_metric('success_count', success_count)\n\n summary = {\n 'success_count': success_count,\n 'source': 'llms_txt' if args.llms_txt else 'url'\n }\n if _source_name:\n summary['source_name'] = _source_name\n\n logger.end(exit_code=exit_code, summary=summary)\n\n except SystemExit:\n raise\n except Exception as e:\n logger.log_error(\"Fatal error in scrape_docs\", error=e)\n exit_code = 1\n logger.end(exit_code=exit_code)\n sys.exit(exit_code)\n\n\nif __name__ == '__main__':\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":47252,"content_sha256":"3a46b8cf1b62cda3cec3d78628d032b0d106521bf3a9a7efb17dfe05b39287c8"},{"filename":"scripts/maintenance/__init__.py","content":"# Maintenance scripts for codex-cli-docs skill\n","content_type":"text/x-python; charset=utf-8","language":"python","size":47,"content_sha256":"a8aa9fe4e968000ce826d1b8553fd94a68958da42cf6d662b19b64bf910b106e"},{"filename":"scripts/management/__init__.py","content":"\"\"\"Index and metadata management for codex-cli-docs skill.\"\"\"\n","content_type":"text/x-python; charset=utf-8","language":"python","size":62,"content_sha256":"b7e6bec80072db1b52ea1a6768ec478a0ca78a11d8f8c690309df6210f112c19"},{"filename":"scripts/utils/__init__.py","content":"\"\"\"Utils package for codex-cli-docs scripts.\"\"\"\n","content_type":"text/x-python; charset=utf-8","language":"python","size":48,"content_sha256":"1e5665aab5ae8e2ff6560dc143a8920cc5594aba458c3dc6b4d181ad54413ddb"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"OpenAI Codex CLI Documentation Skill","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"CRITICAL: Path Doubling Prevention - MANDATORY","type":"text"}]},{"type":"paragraph","content":[{"text":"ABSOLUTE PROHIBITION: NEVER use ","type":"text","marks":[{"type":"strong"}]},{"text":"cd","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" with ","type":"text","marks":[{"type":"strong"}]},{"text":"&&","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" in PowerShell when running scripts from this skill.","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"The Problem:","type":"text","marks":[{"type":"strong"}]},{"text":" If your current working directory is already inside the skill directory, using relative paths causes PowerShell to resolve paths relative to the current directory instead of the repository root, resulting in path doubling.","type":"text"}]},{"type":"paragraph","content":[{"text":"REQUIRED Solutions (choose one):","type":"text","marks":[{"type":"strong"}]}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"ALWAYS use absolute paths","type":"text","marks":[{"type":"strong"}]},{"text":" (recommended)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Use separate commands","type":"text","marks":[{"type":"strong"}]},{"text":" (never ","type":"text"},{"text":"cd","type":"text","marks":[{"type":"code_inline"}]},{"text":" with ","type":"text"},{"text":"&&","type":"text","marks":[{"type":"code_inline"}]},{"text":")","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Run from repository root","type":"text","marks":[{"type":"strong"}]},{"text":" with relative paths","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"NEVER DO THIS:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Chain ","type":"text"},{"text":"cd","type":"text","marks":[{"type":"code_inline"}]},{"text":" with ","type":"text"},{"text":"&&","type":"text","marks":[{"type":"code_inline"}]},{"text":": ","type":"text"},{"text":"cd \u003crelative-path> && python \u003cscript>","type":"text","marks":[{"type":"code_inline"}]},{"text":" causes path doubling","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Assume current directory","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Use relative paths when current dir is inside skill directory","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"CRITICAL: Large File Handling - MANDATORY SCRIPT USAGE","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"ABSOLUTE PROHIBITION: NEVER use read_file tool on the index.yaml file","type":"text"}]},{"type":"paragraph","content":[{"text":"The file exceeds context limits and will cause issues. You MUST use scripts.","type":"text"}]},{"type":"paragraph","content":[{"text":"REQUIRED: ALWAYS use manage_index.py scripts for ANY index.yaml access:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python scripts/management/manage_index.py count\npython scripts/management/manage_index.py list\npython scripts/management/manage_index.py get \u003cdoc_id>\npython scripts/management/manage_index.py verify","type":"text"}]},{"type":"paragraph","content":[{"text":"All scripts automatically handle large files via ","type":"text"},{"text":"index_manager.py","type":"text","marks":[{"type":"code_inline"}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Available Slash Commands","type":"text"}]},{"type":"paragraph","content":[{"text":"Use the consolidated ","type":"text"},{"text":"docs-ops","type":"text","marks":[{"type":"code_inline"}]},{"text":" skill for common workflows:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"/openai-ecosystem:docs-ops scrape","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Scrape Codex CLI documentation from llms.txt sources","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"/openai-ecosystem:docs-ops refresh","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Refresh the local index and metadata without scraping","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"/openai-ecosystem:docs-ops validate","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Validate the index and references for consistency","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"/openai-ecosystem:docs-ops rebuild-index","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Force rebuild the search index","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"/openai-ecosystem:docs-ops clear-cache","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Clear the documentation search cache","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Overview","type":"text"}]},{"type":"paragraph","content":[{"text":"This skill provides automation tooling for OpenAI Codex CLI documentation management. It manages:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Canonical storage","type":"text","marks":[{"type":"strong"}]},{"text":" (encapsulated in skill) - Single source of truth for official docs","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Subsection extraction","type":"text","marks":[{"type":"strong"}]},{"text":" - Token-optimized extracts (60-90% savings)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Drift detection","type":"text","marks":[{"type":"strong"}]},{"text":" - Hash-based validation against upstream sources","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Sync workflows","type":"text","marks":[{"type":"strong"}]},{"text":" - Maintenance automation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Documentation discovery","type":"text","marks":[{"type":"strong"}]},{"text":" - Keyword-based search and doc_id resolution","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Index management","type":"text","marks":[{"type":"strong"}]},{"text":" - Metadata, keywords, tags, aliases for resilient references","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Core value:","type":"text","marks":[{"type":"strong"}]},{"text":" Prevents link rot, enables offline access, optimizes token costs, automates maintenance, and provides resilient doc_id-based references.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"When to Use This Skill","type":"text"}]},{"type":"paragraph","content":[{"text":"This skill should be used when:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Scraping documentation","type":"text","marks":[{"type":"strong"}]},{"text":" - Fetching docs from OpenAI Codex CLI sources","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Finding documentation","type":"text","marks":[{"type":"strong"}]},{"text":" - Searching for docs by keywords, category, or natural language","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Resolving doc references","type":"text","marks":[{"type":"strong"}]},{"text":" - Converting doc_id to file paths","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Managing index metadata","type":"text","marks":[{"type":"strong"}]},{"text":" - Adding keywords, tags, aliases, updating metadata","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Rebuilding index","type":"text","marks":[{"type":"strong"}]},{"text":" - Regenerating index from filesystem (handles renames/moves)","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Workflow Execution Pattern","type":"text"}]},{"type":"paragraph","content":[{"text":"CRITICAL: This section defines HOW to execute operations in this skill.","type":"text","marks":[{"type":"strong"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Delegation Strategy","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Default approach: Delegate to Task agent","type":"text"}]},{"type":"paragraph","content":[{"text":"For ALL scraping, validation, and index operations, delegate execution to a general-purpose Task agent.","type":"text"}]},{"type":"paragraph","content":[{"text":"How to invoke:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Use the Task tool with:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"subagent_type","type":"text","marks":[{"type":"code_inline"}]},{"text":": \"general-purpose\"","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"description","type":"text","marks":[{"type":"code_inline"}]},{"text":": Short 3-5 word description","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"prompt","type":"text","marks":[{"type":"code_inline"}]},{"text":": Full task description with execution instructions","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Execution Pattern","type":"text"}]},{"type":"paragraph","content":[{"text":"Scripts run in FOREGROUND by default. Do NOT background them.","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"When Task agents execute scripts:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Run directly","type":"text","marks":[{"type":"strong"}]},{"text":": ","type":"text"},{"text":"python plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/scrape_docs.py","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Streaming logs","type":"text","marks":[{"type":"strong"}]},{"text":": Scripts emit progress naturally via stdout","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Wait for completion","type":"text","marks":[{"type":"strong"}]},{"text":": Scripts exit when done with exit code","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"NEVER use ","type":"text","marks":[{"type":"strong"}]},{"text":"run_in_background=true","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":": Scripts are designed for foreground execution","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"NEVER poll output","type":"text","marks":[{"type":"strong"}]},{"text":": Streaming logs appear automatically, no BashOutput polling needed","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"NEVER use background jobs","type":"text","marks":[{"type":"strong"}]},{"text":": No ","type":"text"},{"text":"&","type":"text","marks":[{"type":"code_inline"}]},{"text":", no ","type":"text"},{"text":"nohup","type":"text","marks":[{"type":"code_inline"}]},{"text":", no background process management","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Error and Warning Reporting","type":"text"}]},{"type":"paragraph","content":[{"text":"CRITICAL: Report ALL errors, warnings, and issues - never suppress or ignore them.","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"When executing scripts via Task agents:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Report script errors","type":"text","marks":[{"type":"strong"}]},{"text":": Exit codes, exceptions, error messages","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Report warnings","type":"text","marks":[{"type":"strong"}]},{"text":": Deprecation warnings, import issues, configuration problems","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Report unexpected output","type":"text","marks":[{"type":"strong"}]},{"text":": 404s, timeouts, validation failures","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Include context","type":"text","marks":[{"type":"strong"}]},{"text":": What was being executed when the error occurred","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Quick Start","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Refresh Index End-to-End (No Scraping)","type":"text"}]},{"type":"paragraph","content":[{"text":"Use this when you want to rebuild and validate the local index/metadata ","type":"text"},{"text":"without scraping","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"python plugins/openai-ecosystem/skills/codex-cli-docs/scripts/management/refresh_index.py","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scrape All Documentation","type":"text"}]},{"type":"paragraph","content":[{"text":"Use this when the user explicitly wants to ","type":"text"},{"text":"hit the network and scrape docs","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Scrape from configured llms.txt sources\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/scrape_docs.py\n\n# Refresh index after scraping\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/management/refresh_index.py","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Find Documentation","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# Resolve doc_id to file path\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/find_docs.py resolve \u003cdoc_id>\n\n# Search by keywords (default: 25 results)\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/find_docs.py search sandbox tools\n\n# Natural language search\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/find_docs.py query \"how to configure sandbox\"\n\n# List by category\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/find_docs.py category guides\n\n# List by tag\npython plugins/openai-ecosystem/skills/codex-cli-docs/scripts/core/find_docs.py tag cli","type":"text"}]},{"type":"paragraph","content":[{"text":"Search Options:","type":"text","marks":[{"type":"strong"}]}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Option","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Default","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Description","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--limit N","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"25","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Maximum number of results to return","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--no-limit","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"-","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Return all matching results (no limit)","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--min-score N","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"-","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Only return results with relevance score >= N","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--fast","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"-","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Index-only search (skip content grep)","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--json","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"-","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Output results as JSON","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"--verbose","type":"text","marks":[{"type":"code_inline"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"-","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Show relevance scores","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Configuration System","type":"text"}]},{"type":"paragraph","content":[{"text":"The codex-cli-docs skill uses a unified configuration system.","type":"text"}]},{"type":"paragraph","content":[{"text":"Configuration Files:","type":"text","marks":[{"type":"strong"}]}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"config/defaults.yaml","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Central configuration file with all default values","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"config/filtering.yaml","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Content filtering rules","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"config/tag_detection.yaml","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Tag detection patterns","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"references/sources.json","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" - Documentation sources configuration","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Environment Variable Overrides:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"All configuration values can be overridden using environment variables: ","type":"text"},{"text":"CODEX_DOCS_\u003cSECTION>_\u003cKEY>","type":"text","marks":[{"type":"code_inline"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Dependencies","type":"text"}]},{"type":"paragraph","content":[{"text":"Required:","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"text":"pyyaml","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"requests","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"beautifulsoup4","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"markdownify","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"filelock","type":"text","marks":[{"type":"code_inline"}]},{"text":" ","type":"text"},{"text":"Optional (recommended):","type":"text","marks":[{"type":"strong"}]},{"text":" ","type":"text"},{"text":"yake","type":"text","marks":[{"type":"code_inline"}]},{"text":" (for keyword extraction)","type":"text"}]},{"type":"paragraph","content":[{"text":"Python Version:","type":"text","marks":[{"type":"strong"}]},{"text":" Python 3.11+ recommended","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Core Capabilities","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"1. Scraping Documentation","type":"text"}]},{"type":"paragraph","content":[{"text":"Fetch documentation from configured sources using llms.txt format.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"2. Extracting Subsections","type":"text"}]},{"type":"paragraph","content":[{"text":"Extract specific markdown sections for token-optimized responses.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"3. Change Detection","type":"text"}]},{"type":"paragraph","content":[{"text":"Detect documentation drift via 404 checking and hash comparison.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"4. Finding and Resolving Documentation","type":"text"}]},{"type":"paragraph","content":[{"text":"Discover and resolve documentation references using doc_id, keywords, or natural language queries.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"5. Index Management and Maintenance","type":"text"}]},{"type":"paragraph","content":[{"text":"Maintain index metadata, keywords, tags, and rebuild index from filesystem.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Platform-Specific Requirements","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Windows Users","type":"text"}]},{"type":"paragraph","content":[{"text":"MUST use PowerShell (recommended) or prefix Git Bash commands with ","type":"text","marks":[{"type":"strong"}]},{"text":"MSYS_NO_PATHCONV=1","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"Git Bash on Windows converts Unix paths to Windows paths, breaking filter patterns.","type":"text"}]},{"type":"paragraph","content":[{"text":"Example:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"MSYS_NO_PATHCONV=1 python scripts/core/scrape_docs.py --filter \"/docs/\"","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Troubleshooting","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Unicode Encoding Errors","type":"text"}]},{"type":"paragraph","content":[{"text":"Status:","type":"text","marks":[{"type":"strong"}]},{"text":" FIXED - Scripts auto-detect Windows and configure UTF-8 encoding.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"404 Errors During Scraping","type":"text"}]},{"type":"paragraph","content":[{"text":"Status:","type":"text","marks":[{"type":"strong"}]},{"text":" EXPECTED - Some llms.txt entries may reference docs that don't exist yet. Scripts handle gracefully and continue.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Public API","type":"text"}]},{"type":"paragraph","content":[{"text":"The codex-cli-docs skill provides a clean public API for external tools:","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from codex_docs_api import (\n find_document,\n resolve_doc_id,\n get_docs_by_tag,\n get_docs_by_category,\n search_by_keywords,\n get_document_section,\n detect_drift,\n refresh_index\n)\n\n# Natural language search\ndocs = find_document(\"sandbox configuration\")\n\n# Resolve doc_id to metadata\ndoc = resolve_doc_id(\"codex-cli-docs-getting-started\")\n\n# Get docs by tag\ncli_docs = get_docs_by_tag(\"cli\")\n\n# Extract specific section\nsection = get_document_section(\"codex-cli-overview\", \"Installation\")","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Development Mode","type":"text"}]},{"type":"paragraph","content":[{"text":"When developing this plugin locally, you may want changes to go to your dev repo instead of the installed plugin location.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Enabling Dev Mode","type":"text"}]},{"type":"paragraph","content":[{"text":"PowerShell:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"powershell"},"content":[{"text":"$env:CODEX_DOCS_DEV_ROOT = \"D:\\repos\\gh\\melodic\\claude-code-plugins\"","type":"text"}]},{"type":"paragraph","content":[{"text":"Bash/Zsh:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"export CODEX_DOCS_DEV_ROOT=\"/path/to/claude-code-plugins\"","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Verifying Mode","type":"text"}]},{"type":"paragraph","content":[{"text":"When you run any major script (scrape, refresh, rebuild), a mode banner will display:","type":"text"}]},{"type":"paragraph","content":[{"text":"Dev mode:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"text"},"content":[{"text":"[DEV MODE] Using local plugin: D:\\repos\\gh\\melodic\\claude-code-plugins","type":"text"}]},{"type":"paragraph","content":[{"text":"Prod mode:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"text"},"content":[{"text":"[PROD MODE] Using installed skill directory","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Disabling Dev Mode","type":"text"}]},{"type":"paragraph","content":[{"text":"PowerShell:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"powershell"},"content":[{"text":"Remove-Item Env:CODEX_DOCS_DEV_ROOT","type":"text"}]},{"type":"paragraph","content":[{"text":"Bash/Zsh:","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"unset CODEX_DOCS_DEV_ROOT","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Directory Structure","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"text"},"content":[{"text":"codex-cli-docs/\n SKILL.md # This file (public)\n codex_docs_api.py # Public API\n canonical/ # Documentation storage (private)\n index.yaml # Metadata index\n scripts/ # Implementation (private)\n core/ # Scraping, discovery\n management/ # Index management\n maintenance/ # Cleanup, drift detection\n utils/ # Shared utilities\n config/ # Configuration\n defaults.yaml # Default settings\n filtering.yaml # Content filtering\n tag_detection.yaml # Tag patterns\n references/ # Technical documentation (public)\n sources.json # Documentation sources\n .cache/ # Cache storage (inverted index)\n logs/ # Log files","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Source","type":"text"}]},{"type":"paragraph","content":[{"text":"Documentation is scraped from OpenAI Codex CLI llms.txt sources (configured in sources.json).","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Test Scenarios","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scenario 1: Keyword Search","type":"text"}]},{"type":"paragraph","content":[{"text":"Query","type":"text","marks":[{"type":"strong"}]},{"text":": \"Search for sandbox documentation\" ","type":"text"},{"text":"Expected Behavior","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Skill activates on keyword \"documentation\"","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Returns relevant docs from index ","type":"text"},{"text":"Success Criteria","type":"text","marks":[{"type":"strong"}]},{"text":": User receives matching documentation entries","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scenario 2: Natural Language Query","type":"text"}]},{"type":"paragraph","content":[{"text":"Query","type":"text","marks":[{"type":"strong"}]},{"text":": \"How do I configure the Codex CLI sandbox?\" ","type":"text"},{"text":"Expected Behavior","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Skill activates on \"Codex CLI\"","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Uses find_docs.py query command ","type":"text"},{"text":"Success Criteria","type":"text","marks":[{"type":"strong"}]},{"text":": Returns relevant documentation with configuration steps","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Scenario 3: Doc ID Resolution","type":"text"}]},{"type":"paragraph","content":[{"text":"Query","type":"text","marks":[{"type":"strong"}]},{"text":": \"Resolve codex-cli-getting-started\" ","type":"text"},{"text":"Expected Behavior","type":"text","marks":[{"type":"strong"}]},{"text":":","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Resolves doc_id to file path","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Returns document metadata ","type":"text"},{"text":"Success Criteria","type":"text","marks":[{"type":"strong"}]},{"text":": User receives full path and document content","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Version History","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"v1.0.0 (2025-12-15): Initial release - full skill structure adapted from gemini-cli-docs","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Last Updated","type":"text"}]},{"type":"paragraph","content":[{"text":"Date:","type":"text","marks":[{"type":"strong"}]},{"text":" 2025-12-15 ","type":"text"},{"text":"Model:","type":"text","marks":[{"type":"strong"}]},{"text":" claude-opus-4-5-20251101","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Status:","type":"text","marks":[{"type":"strong"}]},{"text":" Initial release - awaiting llms.txt source configuration for scraping.","type":"text"}]}]},"metadata":{"date":"2026-06-05","name":"codex-cli-docs","author":"@skillopedia","source":{"stars":76,"repo_name":"claude-code-plugins","origin_url":"https://github.com/melodic-software/claude-code-plugins/blob/HEAD/plugins/openai-ecosystem/skills/codex-cli-docs/SKILL.md","repo_owner":"melodic-software","body_sha256":"9d885825f7777d410c0c426988a837ebd13b055b0d691e970ccc7826be92af92","cluster_key":"cac019eabb72774ebada70b3220ca6cc31d3b9c5aea88491c74b5fb5ee795a70","clean_bundle":{"format":"clean-skill-bundle-v1","source":"melodic-software/claude-code-plugins/plugins/openai-ecosystem/skills/codex-cli-docs/SKILL.md","attachments":[{"id":"2ab6d969-d7b8-577f-b7fd-56a14187ef8a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2ab6d969-d7b8-577f-b7fd-56a14187ef8a/attachment.md","path":"canonical/developers-openai-com/codex/agent-approvals-security.md","size":17660,"sha256":"96c5c04b113021c342d76b5524fdb15ab3f20010598a34f2851c970801223f2d","contentType":"text/markdown; charset=utf-8"},{"id":"b052cd65-fbe2-5baa-94c2-2f45b9dc31d7","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b052cd65-fbe2-5baa-94c2-2f45b9dc31d7/attachment.md","path":"canonical/developers-openai-com/codex/app-server.md","size":58200,"sha256":"83257db5cc2c71a2dca82de843588fa0cb58a7b98fa47d500cd8f6525d03fe89","contentType":"text/markdown; charset=utf-8"},{"id":"ebef29c0-ff85-5ee1-a544-b9ab6b5522f7","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ebef29c0-ff85-5ee1-a544-b9ab6b5522f7/attachment.md","path":"canonical/developers-openai-com/codex/app.md","size":7550,"sha256":"0dd38487e5924e23eae4b22165aba1d2274ab79e9fe9d344daec90ddeb2ef45d","contentType":"text/markdown; charset=utf-8"},{"id":"d525cfff-0483-52e8-a1c1-7b0978b8cd12","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d525cfff-0483-52e8-a1c1-7b0978b8cd12/attachment.md","path":"canonical/developers-openai-com/codex/app/automations.md","size":9956,"sha256":"1986812a0910256c32f2e48b04dacbc39a777ba52cf76ea29fbeb95b71b69faf","contentType":"text/markdown; charset=utf-8"},{"id":"bb11ec7e-f465-515c-acfe-5616028e7081","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/bb11ec7e-f465-515c-acfe-5616028e7081/attachment.md","path":"canonical/developers-openai-com/codex/app/commands.md","size":5584,"sha256":"0d3c5e9b44eeac139fa56fee22d29db0f4d147693bd94849ecb831d52a286f16","contentType":"text/markdown; charset=utf-8"},{"id":"d4f26042-81c3-59d4-b874-0e751f2b999b","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d4f26042-81c3-59d4-b874-0e751f2b999b/attachment.md","path":"canonical/developers-openai-com/codex/app/features.md","size":11730,"sha256":"f95267f3d662e8b0baf5d9d4e3ab6cf98ee7416755600c96fb6262f980915127","contentType":"text/markdown; charset=utf-8"},{"id":"7c74d507-0dfa-5525-9870-42849719e66d","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/7c74d507-0dfa-5525-9870-42849719e66d/attachment.md","path":"canonical/developers-openai-com/codex/app/local-environments.md","size":2634,"sha256":"825aef4a282bee359f9024e646ae46facc17e2878d1fa6bc45309bcc66f84e7a","contentType":"text/markdown; charset=utf-8"},{"id":"f4fefef4-d781-5cd6-8205-8ba30e132602","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/f4fefef4-d781-5cd6-8205-8ba30e132602/attachment.md","path":"canonical/developers-openai-com/codex/app/review.md","size":3418,"sha256":"f090b1670d6de8ed99cad11e09e29cb3d16ba748fb439c2da8eb3294f7674eca","contentType":"text/markdown; charset=utf-8"},{"id":"2df24936-9b43-50f5-8731-43fafb4ca348","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2df24936-9b43-50f5-8731-43fafb4ca348/attachment.md","path":"canonical/developers-openai-com/codex/app/settings.md","size":2788,"sha256":"79ce412f21defb52cb2017983d2542496fba433c08789e928ccd7815f3eb19f6","contentType":"text/markdown; charset=utf-8"},{"id":"35178354-0480-57b3-b824-3d66ca205a0e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/35178354-0480-57b3-b824-3d66ca205a0e/attachment.md","path":"canonical/developers-openai-com/codex/app/troubleshooting.md","size":5806,"sha256":"03ce08379d9d7bbd9c8119a580b5d94331c98783e7769b0b0b89aa15f1c81be4","contentType":"text/markdown; charset=utf-8"},{"id":"40a3f7db-5562-54dd-ae27-c382538c6be2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/40a3f7db-5562-54dd-ae27-c382538c6be2/attachment.md","path":"canonical/developers-openai-com/codex/app/windows.md","size":9892,"sha256":"2c28d2cae167716413cd5f0b2e1f3dd1f82a98cf9264bfd73bfb64f61eb29aab","contentType":"text/markdown; charset=utf-8"},{"id":"9640af0b-7fd4-57d8-86af-09d1c803c2b6","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/9640af0b-7fd4-57d8-86af-09d1c803c2b6/attachment.md","path":"canonical/developers-openai-com/codex/app/worktrees.md","size":11266,"sha256":"f8197a6b616243c0c03c996727c58482608eb71c81f0ed33c206a9d92535c057","contentType":"text/markdown; charset=utf-8"},{"id":"8bc74004-0d4b-52f1-965e-e260cf16f941","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8bc74004-0d4b-52f1-965e-e260cf16f941/attachment.md","path":"canonical/developers-openai-com/codex/auth.md","size":9870,"sha256":"c5717d30b327422f6ff3f71541c19e70a86290330503f1cbe2da5f14421aaecf","contentType":"text/markdown; charset=utf-8"},{"id":"a26d00c5-d111-5a90-aeda-416bf6a25c5f","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/a26d00c5-d111-5a90-aeda-416bf6a25c5f/attachment.md","path":"canonical/developers-openai-com/codex/autofix-ci.md","size":5896,"sha256":"c65241b4b974440f7adcec777256fffcb6e376740419e5244082a49b6ab2cef2","contentType":"text/markdown; charset=utf-8"},{"id":"5e4d46be-c905-583d-bbae-df6e4cd63476","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/5e4d46be-c905-583d-bbae-df6e4cd63476/attachment.md","path":"canonical/developers-openai-com/codex/changelog.md","size":105901,"sha256":"2c1a142249177a5ffdb2bacfd362efb69e0a8af3ed2998e166feb43d582bc170","contentType":"text/markdown; charset=utf-8"},{"id":"ef136858-d06a-50a7-a48e-c0bab1b91f9a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ef136858-d06a-50a7-a48e-c0bab1b91f9a/attachment.md","path":"canonical/developers-openai-com/codex/cli.md","size":3069,"sha256":"2473ed16399459239950a3e03e5138f4a90784ced62aa786abbb357c923b3184","contentType":"text/markdown; charset=utf-8"},{"id":"dcf4485b-0d69-55a3-851a-3f356120b823","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/dcf4485b-0d69-55a3-851a-3f356120b823/attachment.md","path":"canonical/developers-openai-com/codex/cli/features.md","size":13460,"sha256":"16670692e53b8ace7db183a58f0a1c47fc06483e7610a0964bd22038092c4062","contentType":"text/markdown; charset=utf-8"},{"id":"67486692-45ba-5e67-b411-29c70b706a0a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/67486692-45ba-5e67-b411-29c70b706a0a/attachment.md","path":"canonical/developers-openai-com/codex/cli/reference.md","size":27149,"sha256":"d0e161d9e42c9862210e73d56c71e3f35cf58133d7997bed1021e23b07eef582","contentType":"text/markdown; charset=utf-8"},{"id":"a55cca70-88dc-5a46-b202-4f4dc44cbdf5","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/a55cca70-88dc-5a46-b202-4f4dc44cbdf5/attachment.md","path":"canonical/developers-openai-com/codex/cli/slash-commands.md","size":17636,"sha256":"870a2be570853006818ab319ed03b066809baf14f239cd3151b51f310ce45871","contentType":"text/markdown; charset=utf-8"},{"id":"b60aa200-e1b1-521c-a1ff-285c384809a2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b60aa200-e1b1-521c-a1ff-285c384809a2/attachment.md","path":"canonical/developers-openai-com/codex/cloud.md","size":2254,"sha256":"1b90a789b6a8544ddd3af052eef289f618c4876f4e4c35790d3dcf47115dc42a","contentType":"text/markdown; charset=utf-8"},{"id":"122bb94b-bc93-5af6-8004-45c7b50496b2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/122bb94b-bc93-5af6-8004-45c7b50496b2/attachment.md","path":"canonical/developers-openai-com/codex/cloud/code-review.md","size":2542,"sha256":"3db54f9fc26e711c4569ed5316f3ee2628c25a6a225493f64d9bd49501f63cdd","contentType":"text/markdown; charset=utf-8"},{"id":"20773b1b-2735-5b17-b2cb-c9928c659eb7","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/20773b1b-2735-5b17-b2cb-c9928c659eb7/attachment.md","path":"canonical/developers-openai-com/codex/cloud/environments.md","size":4629,"sha256":"e125e32ae918db67aa8c813d02cf18ce53eefd4604acf7c9d1b2448905fce4ec","contentType":"text/markdown; charset=utf-8"},{"id":"d1e6ec9b-8700-5d8e-ab5c-1968d22aa610","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d1e6ec9b-8700-5d8e-ab5c-1968d22aa610/attachment.md","path":"canonical/developers-openai-com/codex/cloud/internet-access.md","size":4042,"sha256":"8357caea75edd66e43a28eadd52692e1fb61f5c5ac14fdf5ef50853fa6bbef77","contentType":"text/markdown; charset=utf-8"},{"id":"3c19c21a-ec04-5f9e-a0de-b29724be631b","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/3c19c21a-ec04-5f9e-a0de-b29724be631b/attachment.md","path":"canonical/developers-openai-com/codex/community/codex-for-oss.md","size":2130,"sha256":"812b78d74d1dd4ad59b8621d95cdbca8d554ee4fcfe54e1dc65480fcc5b3ad85","contentType":"text/markdown; charset=utf-8"},{"id":"2733ed40-4ef7-55ba-b2eb-deaba669fa61","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2733ed40-4ef7-55ba-b2eb-deaba669fa61/attachment.md","path":"canonical/developers-openai-com/codex/concepts.md","size":3258,"sha256":"ab374f6859101ee58486cfc70fc47c704c46d1275dd28f465206d4d5adf5e4ba","contentType":"text/markdown; charset=utf-8"},{"id":"e1aa325b-9a0d-5519-b7cf-8c521cb042fd","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/e1aa325b-9a0d-5519-b7cf-8c521cb042fd/attachment.md","path":"canonical/developers-openai-com/codex/concepts/customization.md","size":9001,"sha256":"9922e717b676b754fdf3b797c2d42e5afda7662a851cee6ffd2c6a784b180544","contentType":"text/markdown; charset=utf-8"},{"id":"e798804d-034b-50af-8a4a-4537d742695a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/e798804d-034b-50af-8a4a-4537d742695a/attachment.md","path":"canonical/developers-openai-com/codex/concepts/cyber-safety.md","size":4529,"sha256":"4329358c9664e6176bc42455c14c3f14d9be12e64623e2103444e5e20c3abf0e","contentType":"text/markdown; charset=utf-8"},{"id":"8a10ebbf-13e8-5c75-94a7-c29b29fcb65c","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8a10ebbf-13e8-5c75-94a7-c29b29fcb65c/attachment.md","path":"canonical/developers-openai-com/codex/concepts/multi-agents.md","size":3689,"sha256":"a467ebf91e75769f321b36cf6256c883cd88e29d4a5a861ded1bf93f3f7c2484","contentType":"text/markdown; charset=utf-8"},{"id":"bf6e9904-33a6-5c28-bf2a-e3883ca48595","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/bf6e9904-33a6-5c28-bf2a-e3883ca48595/attachment.md","path":"canonical/developers-openai-com/codex/concepts/sandboxing.md","size":5728,"sha256":"49220984dc0bec2f81bbc0bba3b944da607d4a9a9dd516e68104f047b934cf22","contentType":"text/markdown; charset=utf-8"},{"id":"c67cea80-d49e-542f-81a8-4bd6fb2d10c0","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/c67cea80-d49e-542f-81a8-4bd6fb2d10c0/attachment.md","path":"canonical/developers-openai-com/codex/concepts/subagents.md","size":5384,"sha256":"74a48292da02d83a1a5d8c7171ad2ec33ff7169f41d349a5c0ee5af5d9eeee3a","contentType":"text/markdown; charset=utf-8"},{"id":"0abc4695-ba71-5714-bf08-da1dab4b6e1b","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/0abc4695-ba71-5714-bf08-da1dab4b6e1b/attachment.md","path":"canonical/developers-openai-com/codex/config-advanced.md","size":26736,"sha256":"72d3d02d67192ab924d197b7cdcf2fa47fe78e379eca4b92ab0cdc1caf38bcf4","contentType":"text/markdown; charset=utf-8"},{"id":"f0579dcc-ecdd-515d-9fbf-04caadda1ae0","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/f0579dcc-ecdd-515d-9fbf-04caadda1ae0/attachment.md","path":"canonical/developers-openai-com/codex/config-basic.md","size":8800,"sha256":"470c8c71a46b7045c15e94923f29e14eec35b46aabae4fe9e758770fae74aa4c","contentType":"text/markdown; charset=utf-8"},{"id":"116564f6-4de1-5781-bff1-9f02413e0fbf","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/116564f6-4de1-5781-bff1-9f02413e0fbf/attachment.md","path":"canonical/developers-openai-com/codex/config-reference.md","size":43006,"sha256":"57c1255f81d1b5a1b4b1d17d651c7d6243ef985dc816b3888baf4b9c0050ed00","contentType":"text/markdown; charset=utf-8"},{"id":"fbc54ede-eba8-5cc4-ba64-9a9f53bed279","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/fbc54ede-eba8-5cc4-ba64-9a9f53bed279/attachment.md","path":"canonical/developers-openai-com/codex/config-sample.md","size":23086,"sha256":"e07e9b8a7d763415e274b51299009438f5482e96eaaa7553b0cc934c70ec77e1","contentType":"text/markdown; charset=utf-8"},{"id":"8a30e6db-3478-5593-a596-9ab17e81cf8a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8a30e6db-3478-5593-a596-9ab17e81cf8a/attachment.md","path":"canonical/developers-openai-com/codex/custom-prompts.md","size":3451,"sha256":"138f7fb8c0505e4b882db34de0b63c955b6fc0a0e37818257e5d86792ad4d52d","contentType":"text/markdown; charset=utf-8"},{"id":"6e4c004a-15a3-5803-8b7b-621ac059d3c3","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6e4c004a-15a3-5803-8b7b-621ac059d3c3/attachment.md","path":"canonical/developers-openai-com/codex/enterprise.md","size":10182,"sha256":"9c7f962b0994fed778835c80121f53b8ad790ac33f6326dfb3d8986de2176b63","contentType":"text/markdown; charset=utf-8"},{"id":"417578ca-1586-5eba-aafd-1a3613ecf622","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/417578ca-1586-5eba-aafd-1a3613ecf622/attachment.md","path":"canonical/developers-openai-com/codex/enterprise/admin-setup.md","size":23063,"sha256":"581052d8a53cd5b3dc0be2d11875bc558dae751e1ee488819456bb2a681a4cb5","contentType":"text/markdown; charset=utf-8"},{"id":"dca0cfe7-7210-565d-bf37-be095db9edbf","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/dca0cfe7-7210-565d-bf37-be095db9edbf/attachment.md","path":"canonical/developers-openai-com/codex/enterprise/governance.md","size":5413,"sha256":"2cfcf2b7d491721ceffac6352ab2e672aa2f3f9763a669939c0e2f8c6b8bb004","contentType":"text/markdown; charset=utf-8"},{"id":"6848dfd7-ca43-52e2-993f-1db3629a1111","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6848dfd7-ca43-52e2-993f-1db3629a1111/attachment.md","path":"canonical/developers-openai-com/codex/enterprise/managed-configuration.md","size":10544,"sha256":"a10e3261d41f4dbed90d2e439a075099c28ae3b08eacdcc9b24acc046546a822","contentType":"text/markdown; charset=utf-8"},{"id":"5dd6a9da-9289-52b3-8b4b-2c6b3745d366","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/5dd6a9da-9289-52b3-8b4b-2c6b3745d366/attachment.md","path":"canonical/developers-openai-com/codex/exec-policy.md","size":3464,"sha256":"bd6d62235c34294a389c515dc7d2492fb5b9f885cd86eddea513842f1d545452","contentType":"text/markdown; charset=utf-8"},{"id":"6e23338a-6609-5a24-8be6-0eb99ea1c728","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6e23338a-6609-5a24-8be6-0eb99ea1c728/attachment.md","path":"canonical/developers-openai-com/codex/explore.md","size":18717,"sha256":"1270678ae6d1477118bf62ca149a8594f1c02d72884824de63261ef53f99be0b","contentType":"text/markdown; charset=utf-8"},{"id":"d0c3c619-52dd-5543-b854-d29b8658190e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d0c3c619-52dd-5543-b854-d29b8658190e/attachment.md","path":"canonical/developers-openai-com/codex/feature-maturity.md","size":1721,"sha256":"0f13bef77374c6f3a330e1cc7461d30094d7e3be5abc118eb133acab31ff8794","contentType":"text/markdown; charset=utf-8"},{"id":"4722e865-2831-59b9-b277-00a925c3c730","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/4722e865-2831-59b9-b277-00a925c3c730/attachment.md","path":"canonical/developers-openai-com/codex/github-action.md","size":7502,"sha256":"74846fee738b42750169c505fbda2ff3fc88e266e49ab5091227f52071f63f24","contentType":"text/markdown; charset=utf-8"},{"id":"df8fea1e-9c35-56af-b684-19c2b3755c12","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/df8fea1e-9c35-56af-b684-19c2b3755c12/attachment.md","path":"canonical/developers-openai-com/codex/guides/agents-md.md","size":8371,"sha256":"b63580f91bf2775179b906c53a8d0bb2e3f247b9ec7cea40527b49ce01fd7560","contentType":"text/markdown; charset=utf-8"},{"id":"d4ab30ea-f406-5977-bf63-2fd8e89fbf07","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d4ab30ea-f406-5977-bf63-2fd8e89fbf07/attachment.md","path":"canonical/developers-openai-com/codex/guides/agents-sdk.md","size":19559,"sha256":"3ea035ce50659462f4d3680a76325b54abc047348c9d92a8517f2e0901a40209","contentType":"text/markdown; charset=utf-8"},{"id":"af71904f-ab9a-5846-9d24-d3f45ad52437","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/af71904f-ab9a-5846-9d24-d3f45ad52437/attachment.md","path":"canonical/developers-openai-com/codex/guides/api-key.md","size":1831,"sha256":"dcec01482d87f07752a4a1716acfa839511adf79c1af693fe333136dd16a0a43","contentType":"text/markdown; charset=utf-8"},{"id":"be7c66ca-2400-5a40-a9e6-6d901c5247f2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/be7c66ca-2400-5a40-a9e6-6d901c5247f2/attachment.md","path":"canonical/developers-openai-com/codex/guides/autofix-ci.md","size":6643,"sha256":"1ec1b0519df302b1a4dbc1b9d826cbfe58af0c6a6dae08c6abde8b4138e87d12","contentType":"text/markdown; charset=utf-8"},{"id":"6c5d26ee-baae-5428-a6f3-eb8b5d4c4c14","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6c5d26ee-baae-5428-a6f3-eb8b5d4c4c14/attachment.md","path":"canonical/developers-openai-com/codex/guides/build-ai-native-engineering-team.md","size":36826,"sha256":"8ba613f0356e7a18ce6c6a9a6e45cfc7200806682bef409908d70e1c9e16b441","contentType":"text/markdown; charset=utf-8"},{"id":"aedf3d80-850f-56d6-ad6d-68b9b93aaabf","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/aedf3d80-850f-56d6-ad6d-68b9b93aaabf/attachment.md","path":"canonical/developers-openai-com/codex/guides/slash-commands.md","size":9964,"sha256":"01e1f06f1910ce86aea509c8f53be5a1ab700762abe362277cbc4b3c978faaba","contentType":"text/markdown; charset=utf-8"},{"id":"82c838b7-9960-5001-934d-5dea2e612d47","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/82c838b7-9960-5001-934d-5dea2e612d47/attachment.md","path":"canonical/developers-openai-com/codex/ide.md","size":6324,"sha256":"ee76644f26cc7947f8dc617629b349013990a0b97c1560b44567dfc5a66733c4","contentType":"text/markdown; charset=utf-8"},{"id":"58588584-afab-5b5b-8cd3-a0d5fac263f2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/58588584-afab-5b5b-8cd3-a0d5fac263f2/attachment.md","path":"canonical/developers-openai-com/codex/ide/cloud-tasks.md","size":1823,"sha256":"8b0a81ed73ce5ad7fd9595a3e12c4b7006dbc7aa2d86a5afff83d8e4ef4823a0","contentType":"text/markdown; charset=utf-8"},{"id":"3fcf8364-9294-573e-b4c2-a25b02044253","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/3fcf8364-9294-573e-b4c2-a25b02044253/attachment.md","path":"canonical/developers-openai-com/codex/ide/commands.md","size":1870,"sha256":"43b2a4a5d1f56f7a199652cf847e08937d63b09c7246659fd6d1cf7ef7b3f79c","contentType":"text/markdown; charset=utf-8"},{"id":"2de011ea-0c28-5e4d-93f0-59968eaaa331","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2de011ea-0c28-5e4d-93f0-59968eaaa331/attachment.md","path":"canonical/developers-openai-com/codex/ide/features.md","size":5040,"sha256":"4d186fd32b830e1e0fbcca01f70098fea910adf5086a0ba249d4d48695daaf98","contentType":"text/markdown; charset=utf-8"},{"id":"0bc36976-f3a9-5d23-8aaa-4d0b04d89fb9","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/0bc36976-f3a9-5d23-8aaa-4d0b04d89fb9/attachment.md","path":"canonical/developers-openai-com/codex/ide/settings.md","size":2988,"sha256":"097e622cb2f35988a201a60e4a14c386fded34e9644bbc47557eceb45a15e43f","contentType":"text/markdown; charset=utf-8"},{"id":"1da914c6-dbfd-5dbe-a4e6-224394110922","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/1da914c6-dbfd-5dbe-a4e6-224394110922/attachment.md","path":"canonical/developers-openai-com/codex/ide/slash-commands.md","size":1678,"sha256":"9cb632df47634f1b6c86a3c6a3a22ed7da57e45b36bc35ca08759bd7fead1685","contentType":"text/markdown; charset=utf-8"},{"id":"d52e55c6-bb4d-52f1-bb9a-e7fb0a91e8cc","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d52e55c6-bb4d-52f1-bb9a-e7fb0a91e8cc/attachment.md","path":"canonical/developers-openai-com/codex/integrations/github.md","size":3068,"sha256":"4f7dbaa335d6d7f13f506b3a6094877a4e1511c9fcaa314101c940bc5f5c16e4","contentType":"text/markdown; charset=utf-8"},{"id":"44f0b809-7214-5c4a-b716-a04ef77afdd9","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/44f0b809-7214-5c4a-b716-a04ef77afdd9/attachment.md","path":"canonical/developers-openai-com/codex/integrations/linear.md","size":7084,"sha256":"061b1f146c5caa47ed2db521c8b0f77967dc443491ad5d1f17f446fdf4ee043f","contentType":"text/markdown; charset=utf-8"},{"id":"602e847e-0a75-5c78-938a-1e5f8bacd608","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/602e847e-0a75-5c78-938a-1e5f8bacd608/attachment.md","path":"canonical/developers-openai-com/codex/integrations/slack.md","size":4000,"sha256":"97f6d20e08da6f09ebffb0d345e907073d1058a1461f659538c7b0542e1bc431","contentType":"text/markdown; charset=utf-8"},{"id":"648929a3-665f-5419-8db5-71d77a1afa11","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/648929a3-665f-5419-8db5-71d77a1afa11/attachment.md","path":"canonical/developers-openai-com/codex/learn/best-practices.md","size":15182,"sha256":"84938548de3362ff741eefefeb07b9ce85bd2fb83795a889878244e14b3bb251","contentType":"text/markdown; charset=utf-8"},{"id":"bc707d39-5af5-56b6-933b-60196d6f7bcf","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/bc707d39-5af5-56b6-933b-60196d6f7bcf/attachment.md","path":"canonical/developers-openai-com/codex/llms-full.txt.md","size":557536,"sha256":"66fecfdd65e6d1cda07543054c990d2d34c3e6dbe7417179244cf99ff08902b9","contentType":"text/markdown; charset=utf-8"},{"id":"c5e806b4-7e83-58cf-9c45-8f2c5c5571b2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/c5e806b4-7e83-58cf-9c45-8f2c5c5571b2/attachment.md","path":"canonical/developers-openai-com/codex/local-config.md","size":17665,"sha256":"6d49c0f349f4f9f92fcd84c4b9b3ba94a6bd88a5853272046841f2ee89ccda32","contentType":"text/markdown; charset=utf-8"},{"id":"fa263468-e270-5a83-8852-6e09b883ff5d","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/fa263468-e270-5a83-8852-6e09b883ff5d/attachment.md","path":"canonical/developers-openai-com/codex/mcp.md","size":6131,"sha256":"9dfad4bf721ad69ba6e406139cfb4272762fcf31e9f1d42bd688d274849860fb","contentType":"text/markdown; charset=utf-8"},{"id":"6df6ef87-b2c8-5d12-b240-69c90f1dbf24","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6df6ef87-b2c8-5d12-b240-69c90f1dbf24/attachment.md","path":"canonical/developers-openai-com/codex/models.md","size":13251,"sha256":"c46f8be5db7f49cc17b5ecd3083b7045033216afa70e68cb629f18a58ef4dece","contentType":"text/markdown; charset=utf-8"},{"id":"ee0122fc-f8a7-5b80-abb0-5d63bab06dba","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ee0122fc-f8a7-5b80-abb0-5d63bab06dba/attachment.md","path":"canonical/developers-openai-com/codex/multi-agent.md","size":10578,"sha256":"ffb8c4a4d5b93da65bfee3213a960c39aebdb51532340b306f633274fb3de502","contentType":"text/markdown; charset=utf-8"},{"id":"443a65e0-7884-53c9-8cac-4e3453c92f52","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/443a65e0-7884-53c9-8cac-4e3453c92f52/attachment.md","path":"canonical/developers-openai-com/codex/noninteractive.md","size":8802,"sha256":"b6ac8cbed47495787d365747c45b1f355216a09cdfff000f0e8f6e7399e7c0ee","contentType":"text/markdown; charset=utf-8"},{"id":"21c65152-028f-56c4-973d-af989f3078db","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/21c65152-028f-56c4-973d-af989f3078db/attachment.md","path":"canonical/developers-openai-com/codex/open-source.md","size":2878,"sha256":"6c19d38db456143ac336ac8045e15310ef80a6bc513b0cb00cbba2ed60991ff7","contentType":"text/markdown; charset=utf-8"},{"id":"ad225f20-d53b-5431-8c05-af0b5fedd49c","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ad225f20-d53b-5431-8c05-af0b5fedd49c/attachment.md","path":"canonical/developers-openai-com/codex/overview.md","size":2784,"sha256":"7c2eb32e6b1183f1a79ce9da2253f7001451c2691f05e1e7e51febd4aed482ad","contentType":"text/markdown; charset=utf-8"},{"id":"dce86e65-e2a9-5adf-9f04-5de20f267cff","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/dce86e65-e2a9-5adf-9f04-5de20f267cff/attachment.md","path":"canonical/developers-openai-com/codex/pricing.md","size":17953,"sha256":"813b96dbd7e38048f851da79f308663e11ece792ddfe8e0c682763d10eb54fcd","contentType":"text/markdown; charset=utf-8"},{"id":"371e93f7-29c1-5ff1-8aa4-bf3adeafe4cd","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/371e93f7-29c1-5ff1-8aa4-bf3adeafe4cd/attachment.md","path":"canonical/developers-openai-com/codex/prompting.md","size":3544,"sha256":"af2be30168f13bfd7230894b04511977305ab71734ac13761f890c0763ad5f32","contentType":"text/markdown; charset=utf-8"},{"id":"62147b37-2deb-5f79-915b-7bd2a6dd498a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/62147b37-2deb-5f79-915b-7bd2a6dd498a/attachment.md","path":"canonical/developers-openai-com/codex/quickstart.md","size":18538,"sha256":"249b3c6cc58e64bb10dbc77caf7d30b91ee5815be27e25fb90ab085c73225138","contentType":"text/markdown; charset=utf-8"},{"id":"bbf3c0e2-b8d7-5a7a-9f96-f3a14f2ba781","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/bbf3c0e2-b8d7-5a7a-9f96-f3a14f2ba781/attachment.md","path":"canonical/developers-openai-com/codex/rules.md","size":6092,"sha256":"78056f544e245fc0ee58505077a08db76e3f9a13b1f59719dcd66931603d917c","contentType":"text/markdown; charset=utf-8"},{"id":"b952dd30-175e-5aec-96bf-b141903e1a4e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b952dd30-175e-5aec-96bf-b141903e1a4e/attachment.md","path":"canonical/developers-openai-com/codex/sandbox.md","size":1701,"sha256":"bd2ed49b3ac71ffec545949652dcf315c77b061dced5f8a64745728eaaf7e636","contentType":"text/markdown; charset=utf-8"},{"id":"cacdfec5-03a4-560e-937c-810a6cae58d5","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/cacdfec5-03a4-560e-937c-810a6cae58d5/attachment.md","path":"canonical/developers-openai-com/codex/sdk.md","size":1771,"sha256":"e0ca8ae7f8da33bd1e92dd7e0010a7c60f265ed6a7fb68b8c06f0cd8429ed60b","contentType":"text/markdown; charset=utf-8"},{"id":"83d1f25b-f548-5c89-b1dc-42496b6261e6","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/83d1f25b-f548-5c89-b1dc-42496b6261e6/attachment.md","path":"canonical/developers-openai-com/codex/security.md","size":2094,"sha256":"799481925b95ca36d312f690967c0f6bd47dc5386efc2670314b789ecb557aa5","contentType":"text/markdown; charset=utf-8"},{"id":"65580ab4-ecc9-57b7-bb58-e2d3a01b3fb4","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/65580ab4-ecc9-57b7-bb58-e2d3a01b3fb4/attachment.md","path":"canonical/developers-openai-com/codex/security/faq.md","size":6784,"sha256":"0a776e80da2ecf6bf55911d37c5833195478eceb830a3eaa54a53e411bba2b2b","contentType":"text/markdown; charset=utf-8"},{"id":"654b549c-e876-5dbb-89b1-46684ef65f6e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/654b549c-e876-5dbb-89b1-46684ef65f6e/attachment.md","path":"canonical/developers-openai-com/codex/security/setup.md","size":5391,"sha256":"d584851212c78702a673603e7be63475fb4a90375452b40b48e5eb3c56392d1d","contentType":"text/markdown; charset=utf-8"},{"id":"215f00c6-2081-527a-aafc-c66d4fc613d6","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/215f00c6-2081-527a-aafc-c66d4fc613d6/attachment.md","path":"canonical/developers-openai-com/codex/security/threat-model.md","size":2263,"sha256":"3a0f54a47def7b530bff8b53fecff215e0b4a106c93f22d007619ae968ba26c6","contentType":"text/markdown; charset=utf-8"},{"id":"7ad30ad7-7211-5043-bdab-f6038375e575","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/7ad30ad7-7211-5043-bdab-f6038375e575/attachment.md","path":"canonical/developers-openai-com/codex/skills.md","size":7811,"sha256":"bf0332b8122acd05fa4c3eca3fa3911696992b77bc91332ab808a27180e9da2e","contentType":"text/markdown; charset=utf-8"},{"id":"db0bdd92-85a2-55c6-8080-e4ddffc3c085","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/db0bdd92-85a2-55c6-8080-e4ddffc3c085/attachment.md","path":"canonical/developers-openai-com/codex/skills/create-skill.md","size":7637,"sha256":"add0b75a75fec5bc00e03001e891217697de9bc451b29f5e76eea84e08e90bb9","contentType":"text/markdown; charset=utf-8"},{"id":"71907fe9-fb3a-5902-81dd-057d790b1e48","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/71907fe9-fb3a-5902-81dd-057d790b1e48/attachment.md","path":"canonical/developers-openai-com/codex/speed.md","size":1419,"sha256":"5a3aef8af8d523fafe97481085d0417e3303ab7eb76f129ff33a7859743789bb","contentType":"text/markdown; charset=utf-8"},{"id":"ac3c4d7b-388e-5562-881e-98140027191e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ac3c4d7b-388e-5562-881e-98140027191e/attachment.md","path":"canonical/developers-openai-com/codex/subagents.md","size":14425,"sha256":"4d359ff31762f3860292e2d0b62fbfde93ad846fed2dfee2166311d12734cba4","contentType":"text/markdown; charset=utf-8"},{"id":"8ad88479-e52b-51db-a8af-6a4d0c153430","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8ad88479-e52b-51db-a8af-6a4d0c153430/attachment.md","path":"canonical/developers-openai-com/codex/videos.md","size":1533,"sha256":"9a0dd9433386ae63a5ca04becbb7678c4b3224ec7b4ab03fd48144940eaf5cb2","contentType":"text/markdown; charset=utf-8"},{"id":"eb49e47a-3645-509b-bb2b-6e328b59146c","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/eb49e47a-3645-509b-bb2b-6e328b59146c/attachment.md","path":"canonical/developers-openai-com/codex/windows.md","size":6169,"sha256":"11af8f57c022bb471d8a8ab003afc36c8bf33e17553bc9ce2e95e67587b7d17d","contentType":"text/markdown; charset=utf-8"},{"id":"6bab5365-b55c-59a3-8261-fcfe538b7ba8","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6bab5365-b55c-59a3-8261-fcfe538b7ba8/attachment.md","path":"canonical/developers-openai-com/codex/workflows.md","size":11221,"sha256":"7725d20547dedc45978075998d01c740451dbb271996aa8257626ae3cfaba1bb","contentType":"text/markdown; charset=utf-8"},{"id":"0fed6afc-bd6a-5bd8-a329-68099eb4d2d0","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/0fed6afc-bd6a-5bd8-a329-68099eb4d2d0/attachment.json","path":"canonical/index.json","size":397914,"sha256":"4b461ff99577442804977962927fe9f8e9db9ba465c9a92b7898c25164c55133","contentType":"application/json; charset=utf-8"},{"id":"66349f24-f8c7-58d7-80c5-07a0656f9eea","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/66349f24-f8c7-58d7-80c5-07a0656f9eea/attachment.yaml","path":"canonical/index.yaml","size":274127,"sha256":"5280d41f9782587026a423e457e26fc012cd260a529ffb72c32f471d43b580b3","contentType":"application/yaml; charset=utf-8"},{"id":"c9d1578a-32bc-5d00-a368-0c8062372102","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/c9d1578a-32bc-5d00-a368-0c8062372102/attachment.py","path":"codex_docs_api.py","size":14590,"sha256":"34b9ed64e86490e23c6301b8c65bc2b0d578f8acbaef2cd70ab0c099aa5f2d7f","contentType":"text/x-python; charset=utf-8"},{"id":"6e57006b-6605-5ef9-a0f4-795e904ef08f","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6e57006b-6605-5ef9-a0f4-795e904ef08f/attachment.py","path":"config/__init__.py","size":47,"sha256":"ce78d018c4ba2cd9e1ccacb1f12070744766d186cd6df3651cb171ffaab2b416","contentType":"text/x-python; charset=utf-8"},{"id":"0161ed68-b5ce-5110-8453-221bba5606c8","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/0161ed68-b5ce-5110-8453-221bba5606c8/attachment.py","path":"config/config_registry.py","size":14142,"sha256":"69f7c24bf73769f2f58803ece06e67c3c91c3a02af3ccd94dbbd31bb6891902f","contentType":"text/x-python; charset=utf-8"},{"id":"44e992b7-85e0-591d-ac23-6260b0a0963f","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/44e992b7-85e0-591d-ac23-6260b0a0963f/attachment.yaml","path":"config/defaults.yaml","size":7547,"sha256":"64d1d71b60ee4629bbc4de3bec0f6cef50439b6a08f63f8f6f9fe5615b97974b","contentType":"application/yaml; charset=utf-8"},{"id":"6376f3a2-6f13-5aef-a1de-7fee522371e2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6376f3a2-6f13-5aef-a1de-7fee522371e2/attachment.yaml","path":"config/filtering.yaml","size":4661,"sha256":"9798b1938a8d1cb8596b15e34b544b0f12d725c7d5d3da6e5ce3038f4472cb14","contentType":"application/yaml; charset=utf-8"},{"id":"b7210e17-8ad7-5144-b92d-f5ca119047b3","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b7210e17-8ad7-5144-b92d-f5ca119047b3/attachment.yaml","path":"config/tag_detection.yaml","size":3585,"sha256":"d6fc82072f1d291952029039effb52642e8345f035ecbe7820681cef35cc6e8e","contentType":"application/yaml; charset=utf-8"},{"id":"8d20aa45-9774-5d34-8717-738dd0fb469f","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8d20aa45-9774-5d34-8717-738dd0fb469f/attachment.json","path":"references/sources.json","size":443,"sha256":"54ae24aef3fefdab3dda8283d92bc61824323e0d6b8f4e54367a05d469dc5561","contentType":"application/json; charset=utf-8"},{"id":"ec6ddfe9-f6f6-5d99-829d-0f9362fb12bc","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ec6ddfe9-f6f6-5d99-829d-0f9362fb12bc/attachment.py","path":"scripts/__init__.py","size":48,"sha256":"094e1e487f2263b6c2dc53093ab8b8818bf9b8460960d0e80393f21ed489b8d5","contentType":"text/x-python; charset=utf-8"},{"id":"a8c42b81-953b-5257-911c-010607a46c0c","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/a8c42b81-953b-5257-911c-010607a46c0c/attachment.py","path":"scripts/bootstrap.py","size":4205,"sha256":"8965c1dda141257e122380992160e06c0d507635b549dfd39df085ee7ce4f8d7","contentType":"text/x-python; charset=utf-8"},{"id":"2c602e39-57eb-5525-ae2e-92dd5dd780bb","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2c602e39-57eb-5525-ae2e-92dd5dd780bb/attachment.py","path":"scripts/core/__init__.py","size":492,"sha256":"6970c244f6b7c6bb5b1e83931b4fa29a90935e78394763863169b83b09f26c7a","contentType":"text/x-python; charset=utf-8"},{"id":"4dcf74ed-2998-50bf-8b84-8b66143d31f2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/4dcf74ed-2998-50bf-8b84-8b66143d31f2/attachment.py","path":"scripts/core/doc_resolver.py","size":45944,"sha256":"676db90c8792171894843081c6808ee8df079d55f5f728c4f4226aba13d04829","contentType":"text/x-python; charset=utf-8"},{"id":"8c95c880-2ab2-5106-ac7d-0628df802a5b","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/8c95c880-2ab2-5106-ac7d-0628df802a5b/attachment.py","path":"scripts/core/find_docs.py","size":16924,"sha256":"1fa2ca718dc1083101e1cf2f2ae354ef91a3d6ee9b0856c4eeba0b7ac36c3acf","contentType":"text/x-python; charset=utf-8"},{"id":"e79d2124-b347-5017-b86c-b5e9ecd928ad","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/e79d2124-b347-5017-b86c-b5e9ecd928ad/attachment.py","path":"scripts/core/llms_parser.py","size":14839,"sha256":"3294c220b25e7614ff90153f983360dc85fcfa5d512c2966ebf31b256ce41973","contentType":"text/x-python; charset=utf-8"},{"id":"d466fc82-36f2-5f84-aa26-85886a723b33","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d466fc82-36f2-5f84-aa26-85886a723b33/attachment.py","path":"scripts/core/scrape_docs.py","size":47252,"sha256":"3a46b8cf1b62cda3cec3d78628d032b0d106521bf3a9a7efb17dfe05b39287c8","contentType":"text/x-python; charset=utf-8"},{"id":"98467337-7158-5e3f-85d5-a3bb725e768b","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/98467337-7158-5e3f-85d5-a3bb725e768b/attachment.py","path":"scripts/maintenance/__init__.py","size":47,"sha256":"a8aa9fe4e968000ce826d1b8553fd94a68958da42cf6d662b19b64bf910b106e","contentType":"text/x-python; charset=utf-8"},{"id":"d52d0528-fe3f-5a95-b419-968ca1de23a8","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/d52d0528-fe3f-5a95-b419-968ca1de23a8/attachment.py","path":"scripts/maintenance/clear_cache.py","size":5617,"sha256":"ffa40604e86cae46f5ce688440e11b61fb735c7ad3bd1576dd9df3b37fe1cc7d","contentType":"text/x-python; charset=utf-8"},{"id":"81f30f78-d3f6-5f32-8915-15a0601be375","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/81f30f78-d3f6-5f32-8915-15a0601be375/attachment.py","path":"scripts/maintenance/validate_index.py","size":8803,"sha256":"dae8c90312b566cd3ddefb85942c039f7c7ad4c2743725b1524dbd2b8bbd4382","contentType":"text/x-python; charset=utf-8"},{"id":"0c689d49-9ca5-5ba5-8f63-cae648156c3e","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/0c689d49-9ca5-5ba5-8f63-cae648156c3e/attachment.py","path":"scripts/management/__init__.py","size":62,"sha256":"b7e6bec80072db1b52ea1a6768ec478a0ca78a11d8f8c690309df6210f112c19","contentType":"text/x-python; charset=utf-8"},{"id":"b8d6d922-ed85-5c27-931b-be3ac16c0a2a","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b8d6d922-ed85-5c27-931b-be3ac16c0a2a/attachment.py","path":"scripts/management/extract_metadata.py","size":18905,"sha256":"bc13ab0038c38b2df4419fbb06b903e9edc6e4fd2139dc6fe63b62545828ee06","contentType":"text/x-python; charset=utf-8"},{"id":"c79c4638-401f-53a0-9af2-010ba8023f27","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/c79c4638-401f-53a0-9af2-010ba8023f27/attachment.py","path":"scripts/management/generate_report.py","size":5823,"sha256":"9d626438cd838004310fff49597b20c62be75ca20dc2f3f3d8b46a531f037bc2","contentType":"text/x-python; charset=utf-8"},{"id":"1b0009b1-79fa-5ff6-a104-eb70be771da5","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/1b0009b1-79fa-5ff6-a104-eb70be771da5/attachment.py","path":"scripts/management/index_manager.py","size":27699,"sha256":"617b95321b8d6d7b302a912ef68a596d56d31b5f1fdcd89c959b8765ddd00213","contentType":"text/x-python; charset=utf-8"},{"id":"b04d0d52-bd99-51e7-9078-ea8a96be2889","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/b04d0d52-bd99-51e7-9078-ea8a96be2889/attachment.py","path":"scripts/management/manage_index.py","size":18135,"sha256":"1f6920d3c2efb10cdab296f62a258d32c3bfb4235f946504f6cee75c737ace28","contentType":"text/x-python; charset=utf-8"},{"id":"2e5f3fc1-2c6b-5ae9-81d3-a68180f0a63d","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/2e5f3fc1-2c6b-5ae9-81d3-a68180f0a63d/attachment.py","path":"scripts/management/rebuild_index.py","size":15620,"sha256":"c37f89e0f21ef1f7a1987cfc962560eb991cf045ff7efe08186ed98214ede71b","contentType":"text/x-python; charset=utf-8"},{"id":"766de02e-52ae-5221-8e9c-68231a491944","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/766de02e-52ae-5221-8e9c-68231a491944/attachment.py","path":"scripts/management/refresh_index.py","size":8527,"sha256":"b652a596da94b9436d76868a636b392be74213d4367695c5a0f8b7d544afffce","contentType":"text/x-python; charset=utf-8"},{"id":"05a1b168-bbad-5665-8a40-388680b4633c","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/05a1b168-bbad-5665-8a40-388680b4633c/attachment.py","path":"scripts/utils/__init__.py","size":48,"sha256":"1e5665aab5ae8e2ff6560dc143a8920cc5594aba458c3dc6b4d181ad54413ddb","contentType":"text/x-python; charset=utf-8"},{"id":"fd0e138e-bb85-5ca8-9e39-3ef1d4cb1ab9","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/fd0e138e-bb85-5ca8-9e39-3ef1d4cb1ab9/attachment.py","path":"scripts/utils/cache_manager.py","size":12144,"sha256":"a2f5073845a9c18a846f72bfcb6614a5867905faf5b9720f7ef959f6fc3a873e","contentType":"text/x-python; charset=utf-8"},{"id":"4824ac47-4962-58ff-ad98-a8fdf73fc3f2","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/4824ac47-4962-58ff-ad98-a8fdf73fc3f2/attachment.py","path":"scripts/utils/cli_utils.py","size":5419,"sha256":"7a98fb6877b3619e6c3c67f467e0964db410c1a0a0e44c920cc039d687b9fbd2","contentType":"text/x-python; charset=utf-8"},{"id":"ccd0d739-2ff0-55d7-8ec3-825abade9399","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ccd0d739-2ff0-55d7-8ec3-825abade9399/attachment.py","path":"scripts/utils/common_paths.py","size":5547,"sha256":"ae66cd7b1eb3c404cfc64b2c253514e6f9cdf18f36f97450457de4c643b7fba0","contentType":"text/x-python; charset=utf-8"},{"id":"a2cc6357-4ea2-5d73-9b9b-4e91f534f620","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/a2cc6357-4ea2-5d73-9b9b-4e91f534f620/attachment.py","path":"scripts/utils/config_helpers.py","size":15914,"sha256":"ce63ad780884c6e827f667a55ee18a1ad697d919d411af844ad97c56fb4b5d67","contentType":"text/x-python; charset=utf-8"},{"id":"37e27982-4b11-51da-a26f-b7a1f0ef0cc0","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/37e27982-4b11-51da-a26f-b7a1f0ef0cc0/attachment.py","path":"scripts/utils/config_loader.py","size":9474,"sha256":"dadb2a89856d52ff121ed9a2b1574ac6326415c24f395a675ad061a5fae8d741","contentType":"text/x-python; charset=utf-8"},{"id":"39eec973-8650-57b8-9a1c-ef1d29b5c3f7","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/39eec973-8650-57b8-9a1c-ef1d29b5c3f7/attachment.py","path":"scripts/utils/constants.py","size":1241,"sha256":"34fe56c94e46f5ccc64988f7e5d758d9684afd84daf9eb315f5485d272d13305","contentType":"text/x-python; charset=utf-8"},{"id":"fd5bc1ae-201d-5115-bcd5-53cf08a3e0a6","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/fd5bc1ae-201d-5115-bcd5-53cf08a3e0a6/attachment.py","path":"scripts/utils/dev_mode.py","size":8547,"sha256":"4df7185a5c195b5fd17643bc524d2062ca59341d4c1fd704bc6962d27410d821","contentType":"text/x-python; charset=utf-8"},{"id":"e28d8981-cba1-5047-aae4-a08403f990f7","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/e28d8981-cba1-5047-aae4-a08403f990f7/attachment.py","path":"scripts/utils/http_utils.py","size":24196,"sha256":"2b055d250a5d7180b3eaff8373dbfb3914818f7e69a9513bb31505d2bb97159b","contentType":"text/x-python; charset=utf-8"},{"id":"ed784fce-42a3-5622-9ff7-751a7ce28fed","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/ed784fce-42a3-5622-9ff7-751a7ce28fed/attachment.py","path":"scripts/utils/logging_utils.py","size":41446,"sha256":"c9e5f9e152e56a0965e84e403245da58d1fe469b1b6185781e0693c8360fdde8","contentType":"text/x-python; charset=utf-8"},{"id":"01d0d0ef-8757-59a6-b2e7-2088650c6eac","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/01d0d0ef-8757-59a6-b2e7-2088650c6eac/attachment.py","path":"scripts/utils/metadata_utils.py","size":1833,"sha256":"fdf7b2358a890e15a4e52fd8db7f756588224fda5c48e1d5dfe968520a011e63","contentType":"text/x-python; charset=utf-8"},{"id":"6e723bee-9cd5-5efa-aafe-7ac9debd5742","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6e723bee-9cd5-5efa-aafe-7ac9debd5742/attachment.py","path":"scripts/utils/path_config.py","size":5299,"sha256":"278a7079c192ce1db42c6edc79424887c310d273774844537839d66024c6e761","contentType":"text/x-python; charset=utf-8"},{"id":"6aa55197-7a8e-5aad-801c-66038b2cc0da","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/6aa55197-7a8e-5aad-801c-66038b2cc0da/attachment.py","path":"scripts/utils/script_utils.py","size":8076,"sha256":"8c40dbc6692ea558b562c18fdcb974b04bb5e28ec65e333646368eb0d6d94840","contentType":"text/x-python; charset=utf-8"},{"id":"3bb3a060-a521-54df-a546-1193680b4f0f","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/3bb3a060-a521-54df-a546-1193680b4f0f/attachment.py","path":"scripts/utils/search_constants.py","size":4735,"sha256":"fef1f2e061a0601f35e3436892af8ac3d314413f3fadc7d1e6c6a2af56f10c68","contentType":"text/x-python; charset=utf-8"}],"bundle_sha256":"8a0849212551e353351eb1132ca4a73219a1a9913a3deee6fd9f4c385ab0cd37","attachment_count":124,"text_attachments":124,"attachment_storage":"skillopedia-attachments-v1","binary_attachments":0,"excluded_attachments":[]},"cluster_size":1,"skill_md_path":"plugins/openai-ecosystem/skills/codex-cli-docs/SKILL.md","import_metadata":{"date":"2026-06-05","author":"@skillopedia","version":"v1","category":"browser-automation-scraping","category_label":"Browser"},"exact_dupes_collapsed_into_this":0},"version":"v1","category":"browser-automation-scraping","import_tag":"clean-skills-v1","description":"Single source of truth and librarian for ALL OpenAI Codex CLI documentation. Manages local documentation storage, scraping, discovery, and resolution. Use when finding, locating, searching, or resolving Codex CLI documentation; discovering docs by keywords, category, tags, or natural language queries; scraping from llms.txt; managing index metadata (keywords, tags, aliases); or rebuilding index from filesystem. Run scripts to scrape, find, and resolve documentation. Handles doc_id resolution, keyword search, natural language queries, category/tag filtering, alias resolution, llms.txt parsing, markdown subsection extraction for internal use, hash-based drift detection, and comprehensive index maintenance.","allowed-tools":"Read, Glob, Grep, Bash"}},"renderedAt":1782979218925}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.