skill-tester — Skillopedia

Skill Tester Tier : POWERFUL · Category : Engineering Quality Assurance · Dependencies : None (Python stdlib only) Meta-skill that validates, tests, and scores skills in this repository. Four tools, run from the repo root with full paths: 1. — structure + documentation compliance 2. — Python script syntax/imports/runtime/output testing 3. — multi-dimensional scoring with letter grade 4. — security posture scoring (also available via ) Scope note: this skill's tier line-count minimums measure legacy skills. For authoring new skills, (SKILL.md under 100 lines, Matt Pocock doctrine) is the bindi…

, content, re.MULTILINE | re.IGNORECASE):\n present_required += 1\n \n score += (present_required / len(required_sections)) * 15\n \n # Recommended sections (10 points)\n present_recommended = 0\n for section in recommended_sections:\n if re.search(rf'^#+\\s*{re.escape(section)}\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

, content, re.MULTILINE | re.IGNORECASE):\n present_recommended += 1\n \n score += (present_recommended / len(recommended_sections)) * 10\n \n return score\n \n def _score_content_depth(self, content: str) -> float:\n \"\"\"Score content depth and technical detail\"\"\"\n score = 0\n \n # Code examples (8 points)\n code_blocks = len(re.findall(r'```[\\w]*\\n.*?\\n```', content, re.DOTALL))\n score += min(code_blocks * 2, 8)\n \n # Technical depth indicators (8 points)\n depth_indicators = ['API', 'algorithm', 'architecture', 'implementation', 'performance', \n 'scalability', 'security', 'integration', 'configuration', 'parameters']\n depth_score = sum(1 for indicator in depth_indicators if indicator.lower() in content.lower())\n score += min(depth_score * 0.8, 8)\n \n # Usage examples (9 points)\n example_patterns = [r'Example:', r'Usage:', r'```bash', r'```python', r'```yaml']\n example_count = sum(len(re.findall(pattern, content, re.IGNORECASE)) for pattern in example_patterns)\n score += min(example_count * 1.5, 9)\n \n return score\n \n def _score_readme(self, dimension: QualityDimension):\n \"\"\"Score README.md quality\"\"\"\n readme_path = self.skill_path / \"README.md\"\n \n if not readme_path.exists():\n dimension.add_score(\"readme_existence\", 10, 25, \"README.md exists (partial credit)\")\n dimension.add_suggestion(\"Create README.md with usage instructions\")\n return\n \n try:\n content = readme_path.read_text(encoding='utf-8')\n \n # Length and substance\n if len(content.strip()) >= 1000:\n length_score = 25\n elif len(content.strip()) >= 500:\n length_score = 20\n elif len(content.strip()) >= 200:\n length_score = 15\n else:\n length_score = 10\n \n dimension.add_score(\"readme_quality\", length_score, 25,\n f\"README.md content quality ({len(content)} characters)\")\n \n if len(content.strip()) \u003c 500:\n dimension.add_suggestion(\"Expand README.md with more detailed usage examples\")\n \n except Exception:\n dimension.add_score(\"readme_readable\", 5, 25, \"README.md exists but has issues\")\n \n def _score_references(self, dimension: QualityDimension):\n \"\"\"Score reference documentation quality\"\"\"\n references_dir = self.skill_path / \"references\"\n \n if not references_dir.exists():\n dimension.add_score(\"references_existence\", 0, 25, \"No references directory\")\n dimension.add_suggestion(\"Add references directory with documentation\")\n return\n \n ref_files = list(references_dir.glob(\"*.md\")) + list(references_dir.glob(\"*.txt\"))\n \n if not ref_files:\n dimension.add_score(\"references_content\", 5, 25, \"References directory empty\")\n dimension.add_suggestion(\"Add reference documentation files\")\n return\n \n # Score based on number and quality of reference files\n score = min(len(ref_files) * 5, 20) # Up to 20 points for multiple files\n \n # Bonus for substantial content\n total_content = 0\n for ref_file in ref_files:\n try:\n content = ref_file.read_text(encoding='utf-8')\n total_content += len(content.strip())\n except:\n continue\n \n if total_content >= 2000:\n score += 5 # Bonus for substantial reference content\n \n dimension.add_score(\"references_quality\", score, 25, \n f\"References: {len(ref_files)} files, {total_content} chars\")\n \n def _score_examples(self, dimension: QualityDimension):\n \"\"\"Score examples and usage clarity\"\"\"\n score = 0\n \n # Look for example files in various locations\n example_locations = [\"examples\", \"assets\", \"scripts\"]\n example_files = []\n \n for location in example_locations:\n location_path = self.skill_path / location\n if location_path.exists():\n example_files.extend(location_path.glob(\"*example*\"))\n example_files.extend(location_path.glob(\"*sample*\"))\n example_files.extend(location_path.glob(\"*demo*\"))\n \n # Score based on example availability\n if len(example_files) >= 3:\n score = 25\n elif len(example_files) >= 2:\n score = 20\n elif len(example_files) >= 1:\n score = 15\n else:\n score = 10\n dimension.add_suggestion(\"Add more usage examples and sample files\")\n \n dimension.add_score(\"examples_availability\", score, 25,\n f\"Found {len(example_files)} example/sample files\")\n \n def _score_code_quality(self, weight: float = 0.25):\n \"\"\"Score code quality\"\"\"\n self.log_verbose(\"Scoring code quality...\")\n \n dimension = QualityDimension(\"Code Quality\", weight, \"Quality of Python scripts and implementation\")\n \n scripts_dir = self.skill_path / \"scripts\"\n if not scripts_dir.exists():\n dimension.add_score(\"scripts_existence\", 0, 100, \"No scripts directory\")\n dimension.add_suggestion(\"Create scripts directory with Python files\")\n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n return\n \n python_files = list(scripts_dir.glob(\"*.py\"))\n if not python_files:\n dimension.add_score(\"python_scripts\", 0, 100, \"No Python scripts found\")\n dimension.add_suggestion(\"Add Python scripts to scripts directory\")\n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n return\n \n # Score script complexity and quality\n self._score_script_complexity(python_files, dimension)\n \n # Score error handling\n self._score_error_handling(python_files, dimension)\n \n # Score code structure\n self._score_code_structure(python_files, dimension)\n \n # Score output format support\n self._score_output_support(python_files, dimension)\n \n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n \n def _score_script_complexity(self, python_files: List[Path], dimension: QualityDimension):\n \"\"\"Score script complexity and sophistication\"\"\"\n total_complexity = 0\n script_count = len(python_files)\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n \n # Count lines of code (excluding empty lines and comments)\n lines = content.split('\\n')\n loc = len([line for line in lines if line.strip() and not line.strip().startswith('#')])\n \n # Score based on LOC\n if loc >= 800:\n complexity_score = 25\n elif loc >= 500:\n complexity_score = 20\n elif loc >= 300:\n complexity_score = 15\n elif loc >= 100:\n complexity_score = 10\n else:\n complexity_score = 5\n \n total_complexity += complexity_score\n \n except Exception:\n continue\n \n avg_complexity = total_complexity / script_count if script_count > 0 else 0\n dimension.add_score(\"script_complexity\", avg_complexity, 25,\n f\"Average script complexity across {script_count} scripts\")\n \n if avg_complexity \u003c 15:\n dimension.add_suggestion(\"Consider expanding scripts with more functionality\")\n \n def _score_error_handling(self, python_files: List[Path], dimension: QualityDimension):\n \"\"\"Score error handling quality\"\"\"\n total_error_score = 0\n script_count = len(python_files)\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n error_score = 0\n \n # Check for try/except blocks\n try_count = content.count('try:')\n error_score += min(try_count * 5, 15) # Up to 15 points for try/except\n \n # Check for specific exception handling\n exception_types = ['Exception', 'ValueError', 'FileNotFoundError', 'KeyError', 'TypeError']\n for exc_type in exception_types:\n if exc_type in content:\n error_score += 2 # 2 points per specific exception type\n \n # Check for logging or error reporting\n if any(indicator in content for indicator in ['print(', 'logging.', 'sys.stderr']):\n error_score += 5 # 5 points for error reporting\n \n total_error_score += min(error_score, 25) # Cap at 25 per script\n \n except Exception:\n continue\n \n avg_error_score = total_error_score / script_count if script_count > 0 else 0\n dimension.add_score(\"error_handling\", avg_error_score, 25,\n f\"Error handling quality across {script_count} scripts\")\n \n if avg_error_score \u003c 15:\n dimension.add_suggestion(\"Improve error handling with try/except blocks and meaningful error messages\")\n \n def _score_code_structure(self, python_files: List[Path], dimension: QualityDimension):\n \"\"\"Score code structure and organization\"\"\"\n total_structure_score = 0\n script_count = len(python_files)\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n structure_score = 0\n \n # Check for functions and classes\n function_count = content.count('def ')\n class_count = content.count('class ')\n \n structure_score += min(function_count * 2, 10) # Up to 10 points for functions\n structure_score += min(class_count * 3, 9) # Up to 9 points for classes\n \n # Check for docstrings\n docstring_patterns = ['\"\"\"', \"'''\", 'def.*:\\n.*\"\"\"', 'class.*:\\n.*\"\"\"']\n for pattern in docstring_patterns:\n if re.search(pattern, content):\n structure_score += 1 # 1 point per docstring indicator\n \n # Check for if __name__ == \"__main__\"\n if 'if __name__ == \"__main__\"' in content:\n structure_score += 3\n \n # Check for imports organization\n if content.lstrip().startswith(('import ', 'from ')):\n structure_score += 2 # Imports at top\n \n total_structure_score += min(structure_score, 25)\n \n except Exception:\n continue\n \n avg_structure_score = total_structure_score / script_count if script_count > 0 else 0\n dimension.add_score(\"code_structure\", avg_structure_score, 25,\n f\"Code structure quality across {script_count} scripts\")\n \n if avg_structure_score \u003c 15:\n dimension.add_suggestion(\"Improve code structure with more functions, classes, and documentation\")\n \n def _score_output_support(self, python_files: List[Path], dimension: QualityDimension):\n \"\"\"Score output format support\"\"\"\n total_output_score = 0\n script_count = len(python_files)\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n output_score = 0\n \n # Check for JSON support\n if any(indicator in content for indicator in ['json.dump', 'json.load', '--json']):\n output_score += 12 # JSON support\n \n # Check for formatted output\n if any(indicator in content for indicator in ['print(f\"', 'print(\"', '.format(', 'f\"']):\n output_score += 8 # Human-readable output\n \n # Check for argparse help\n if '--help' in content or 'add_help=' in content:\n output_score += 5 # Help functionality\n \n total_output_score += min(output_score, 25)\n \n except Exception:\n continue\n \n avg_output_score = total_output_score / script_count if script_count > 0 else 0\n dimension.add_score(\"output_support\", avg_output_score, 25,\n f\"Output format support across {script_count} scripts\")\n \n if avg_output_score \u003c 15:\n dimension.add_suggestion(\"Add support for both JSON and human-readable output formats\")\n \n def _score_completeness(self, weight: float = 0.25):\n \"\"\"Score completeness\"\"\"\n self.log_verbose(\"Scoring completeness...\")\n \n dimension = QualityDimension(\"Completeness\", weight, \"Completeness of required components and assets\")\n \n # Score directory structure\n self._score_directory_structure(dimension)\n \n # Score asset availability\n self._score_assets(dimension)\n \n # Score expected outputs\n self._score_expected_outputs(dimension)\n \n # Score test coverage\n self._score_test_coverage(dimension)\n \n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n \n def _score_directory_structure(self, dimension: QualityDimension):\n \"\"\"Score directory structure completeness\"\"\"\n required_dirs = [\"scripts\"]\n recommended_dirs = [\"assets\", \"references\", \"expected_outputs\"]\n \n score = 0\n \n # Required directories (15 points)\n for dir_name in required_dirs:\n if (self.skill_path / dir_name).exists():\n score += 15 / len(required_dirs)\n \n # Recommended directories (10 points)\n present_recommended = 0\n for dir_name in recommended_dirs:\n if (self.skill_path / dir_name).exists():\n present_recommended += 1\n \n score += (present_recommended / len(recommended_dirs)) * 10\n \n dimension.add_score(\"directory_structure\", score, 25,\n f\"Directory structure completeness\")\n \n missing_recommended = [d for d in recommended_dirs if not (self.skill_path / d).exists()]\n if missing_recommended:\n dimension.add_suggestion(f\"Add recommended directories: {', '.join(missing_recommended)}\")\n \n def _score_assets(self, dimension: QualityDimension):\n \"\"\"Score asset availability and quality\"\"\"\n assets_dir = self.skill_path / \"assets\"\n \n if not assets_dir.exists():\n dimension.add_score(\"assets_existence\", 5, 25, \"Assets directory missing\")\n dimension.add_suggestion(\"Create assets directory with sample data\")\n return\n \n asset_files = [f for f in assets_dir.rglob(\"*\") if f.is_file()]\n \n if not asset_files:\n dimension.add_score(\"assets_content\", 10, 25, \"Assets directory empty\")\n dimension.add_suggestion(\"Add sample data files to assets directory\")\n return\n \n # Score based on number and diversity of assets\n score = min(len(asset_files) * 3, 20) # Up to 20 points for multiple assets\n \n # Bonus for diverse file types\n extensions = set(f.suffix.lower() for f in asset_files if f.suffix)\n if len(extensions) >= 3:\n score += 5 # Bonus for file type diversity\n \n dimension.add_score(\"assets_quality\", score, 25,\n f\"Assets: {len(asset_files)} files, {len(extensions)} types\")\n \n def _score_expected_outputs(self, dimension: QualityDimension):\n \"\"\"Score expected outputs availability\"\"\"\n expected_dir = self.skill_path / \"expected_outputs\"\n \n if not expected_dir.exists():\n dimension.add_score(\"expected_outputs\", 10, 25, \"Expected outputs directory missing\")\n dimension.add_suggestion(\"Add expected_outputs directory with sample results\")\n return\n \n output_files = [f for f in expected_dir.rglob(\"*\") if f.is_file()]\n \n if len(output_files) >= 3:\n score = 25\n elif len(output_files) >= 2:\n score = 20\n elif len(output_files) >= 1:\n score = 15\n else:\n score = 10\n dimension.add_suggestion(\"Add expected output files for testing\")\n \n dimension.add_score(\"expected_outputs\", score, 25,\n f\"Expected outputs: {len(output_files)} files\")\n \n def _score_test_coverage(self, dimension: QualityDimension):\n \"\"\"Score test coverage and validation\"\"\"\n # This is a simplified scoring - in a more sophisticated system,\n # this would integrate with actual test runners\n \n score = 15 # Base score for having a structure\n \n # Check for test-related files\n test_indicators = [\"test\", \"spec\", \"check\"]\n test_files = []\n \n for indicator in test_indicators:\n test_files.extend(self.skill_path.rglob(f\"*{indicator}*\"))\n \n if test_files:\n score += 10 # Bonus for test files\n \n dimension.add_score(\"test_coverage\", score, 25,\n f\"Test coverage indicators: {len(test_files)} files\")\n \n if not test_files:\n dimension.add_suggestion(\"Add test files or validation scripts\")\n \n def _score_usability(self, weight: float = 0.25):\n \"\"\"Score usability\"\"\"\n self.log_verbose(\"Scoring usability...\")\n \n dimension = QualityDimension(\"Usability\", weight, \"Ease of use and user experience\")\n \n # Score installation simplicity\n self._score_installation(dimension)\n \n # Score usage clarity\n self._score_usage_clarity(dimension)\n \n # Score help and documentation accessibility\n self._score_help_accessibility(dimension)\n \n # Score practical examples\n self._score_practical_examples(dimension)\n \n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n \n def _score_installation(self, dimension: QualityDimension):\n \"\"\"Score installation simplicity\"\"\"\n # Check for installation complexity indicators\n score = 25 # Start with full points for standard library only approach\n \n # Check for requirements.txt or setup.py (would reduce score)\n if (self.skill_path / \"requirements.txt\").exists():\n score -= 5 # Minor penalty for external dependencies\n dimension.add_suggestion(\"Consider removing external dependencies for easier installation\")\n \n if (self.skill_path / \"setup.py\").exists():\n score -= 3 # Minor penalty for complex setup\n \n dimension.add_score(\"installation_simplicity\", max(score, 15), 25,\n \"Installation complexity assessment\")\n \n def _score_usage_clarity(self, dimension: QualityDimension):\n \"\"\"Score usage clarity\"\"\"\n score = 0\n \n # Check README for usage instructions\n readme_path = self.skill_path / \"README.md\"\n if readme_path.exists():\n try:\n content = readme_path.read_text(encoding='utf-8').lower()\n if 'usage' in content or 'how to' in content:\n score += 10\n if 'example' in content:\n score += 5\n except:\n pass\n \n # Check scripts for help text quality\n scripts_dir = self.skill_path / \"scripts\"\n if scripts_dir.exists():\n python_files = list(scripts_dir.glob(\"*.py\"))\n help_quality = 0\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n if 'argparse' in content and 'help=' in content:\n help_quality += 2\n except:\n continue\n \n score += min(help_quality, 10) # Up to 10 points for help text\n \n dimension.add_score(\"usage_clarity\", score, 25, \"Usage instructions and help quality\")\n \n if score \u003c 15:\n dimension.add_suggestion(\"Improve usage documentation and help text\")\n \n def _score_help_accessibility(self, dimension: QualityDimension):\n \"\"\"Score help and documentation accessibility\"\"\"\n score = 0\n \n # Check for comprehensive help in scripts\n scripts_dir = self.skill_path / \"scripts\"\n if scripts_dir.exists():\n python_files = list(scripts_dir.glob(\"*.py\"))\n \n for script_path in python_files:\n try:\n content = script_path.read_text(encoding='utf-8')\n \n # Check for detailed help text\n if 'epilog=' in content or 'description=' in content:\n score += 5 # Detailed help\n \n # Check for examples in help\n if 'examples:' in content.lower() or 'example:' in content.lower():\n score += 3 # Examples in help\n \n except:\n continue\n \n # Check for documentation files\n doc_files = list(self.skill_path.glob(\"*.md\"))\n if len(doc_files) >= 2:\n score += 5 # Multiple documentation files\n \n dimension.add_score(\"help_accessibility\", min(score, 25), 25,\n \"Help and documentation accessibility\")\n \n if score \u003c 15:\n dimension.add_suggestion(\"Add more comprehensive help text and documentation\")\n \n def _score_practical_examples(self, dimension: QualityDimension):\n \"\"\"Score practical examples quality\"\"\"\n score = 0\n \n # Look for example files\n example_patterns = [\"*example*\", \"*sample*\", \"*demo*\", \"*tutorial*\"]\n example_files = []\n \n for pattern in example_patterns:\n example_files.extend(self.skill_path.rglob(pattern))\n \n # Score based on example availability and quality\n if len(example_files) >= 5:\n score = 25\n elif len(example_files) >= 3:\n score = 20\n elif len(example_files) >= 2:\n score = 15\n elif len(example_files) >= 1:\n score = 10\n else:\n score = 5\n dimension.add_suggestion(\"Add more practical examples and sample files\")\n \n dimension.add_score(\"practical_examples\", score, 25,\n f\"Practical examples: {len(example_files)} files\")\n\n def _score_security(self, weight: float = 0.20):\n \"\"\"Score security quality\"\"\"\n self.log_verbose(\"Scoring security quality...\")\n \n dimension = QualityDimension(\"Security\", weight, \"Security practices and vulnerability prevention\")\n \n # Find Python scripts\n python_files = list(self.skill_path.rglob(\"*.py\"))\n \n # Filter out test files and __pycache__\n python_files = [f for f in python_files \n if \"__pycache__\" not in str(f) and \"test_\" not in f.name]\n \n if not python_files:\n dimension.add_score(\"scripts_existence\", 25, 25, \n \"No scripts directory - no script security concerns\")\n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n return\n \n # Use SecurityScorer module\n try:\n scorer = SecurityScorer(python_files, verbose=self.verbose)\n result = scorer.get_overall_score()\n \n # Extract scores from SecurityScorer result\n sensitive_data_score = result.get(\"sensitive_data_exposure\", {}).get(\"score\", 0)\n file_ops_score = result.get(\"safe_file_operations\", {}).get(\"score\", 0)\n command_injection_score = result.get(\"command_injection_prevention\", {}).get(\"score\", 0)\n input_validation_score = result.get(\"input_validation\", {}).get(\"score\", 0)\n \n dimension.add_score(\"sensitive_data_exposure\", sensitive_data_score, 25,\n \"Detection and prevention of hardcoded credentials\")\n dimension.add_score(\"safe_file_operations\", file_ops_score, 25,\n \"Prevention of path traversal vulnerabilities\")\n dimension.add_score(\"command_injection_prevention\", command_injection_score, 25,\n \"Prevention of command injection vulnerabilities\")\n dimension.add_score(\"input_validation\", input_validation_score, 25,\n \"Quality of input validation and error handling\")\n \n # Add suggestions from SecurityScorer\n for issue in result.get(\"issues\", []):\n dimension.add_suggestion(issue)\n \n except Exception as e:\n self.log_verbose(f\"Security scoring failed: {str(e)}\")\n dimension.add_score(\"security_error\", 0, 100, f\"Security scoring failed: {str(e)}\")\n dimension.add_suggestion(\"Fix security scoring module integration\")\n \n dimension.calculate_final_score()\n self.report.add_dimension(dimension)\n\n\nclass QualityReportFormatter:\n \"\"\"Formats quality reports for output\"\"\"\n \n @staticmethod\n def format_json(report: QualityReport) -> str:\n \"\"\"Format report as JSON\"\"\"\n return json.dumps({\n \"skill_path\": report.skill_path,\n \"timestamp\": report.timestamp,\n \"overall_score\": round(report.overall_score, 1),\n \"letter_grade\": report.letter_grade,\n \"tier_recommendation\": report.tier_recommendation,\n \"summary_stats\": report.summary_stats,\n \"dimensions\": {\n name: {\n \"name\": dim.name,\n \"weight\": dim.weight,\n \"score\": round(dim.score, 1),\n \"description\": dim.description,\n \"details\": dim.details,\n \"suggestions\": dim.suggestions\n }\n for name, dim in report.dimensions.items()\n },\n \"improvement_roadmap\": report.improvement_roadmap\n }, indent=2)\n \n @staticmethod\n def format_human_readable(report: QualityReport, detailed: bool = False) -> str:\n \"\"\"Format report as human-readable text\"\"\"\n lines = []\n lines.append(\"=\" * 70)\n lines.append(\"SKILL QUALITY ASSESSMENT REPORT\")\n lines.append(\"=\" * 70)\n lines.append(f\"Skill: {report.skill_path}\")\n lines.append(f\"Timestamp: {report.timestamp}\")\n lines.append(f\"Overall Score: {report.overall_score:.1f}/100 ({report.letter_grade})\")\n lines.append(f\"Recommended Tier: {report.tier_recommendation}\")\n lines.append(\"\")\n \n # Dimension scores\n lines.append(\"QUALITY DIMENSIONS:\")\n for name, dimension in report.dimensions.items():\n lines.append(f\" {name}: {dimension.score:.1f}/100 ({dimension.weight * 100:.0f}% weight)\")\n if detailed and dimension.details:\n for component, details in dimension.details.items():\n lines.append(f\" • {component}: {details['score']:.1f}/{details['max_score']} - {details['details']}\")\n lines.append(\"\")\n \n # Summary statistics\n if report.summary_stats:\n lines.append(\"SUMMARY STATISTICS:\")\n lines.append(f\" Highest Dimension: {report.summary_stats['highest_dimension']}\")\n lines.append(f\" Lowest Dimension: {report.summary_stats['lowest_dimension']}\")\n lines.append(f\" Dimensions Above 70%: {report.summary_stats['dimensions_above_70']}\")\n lines.append(f\" Dimensions Below 50%: {report.summary_stats['dimensions_below_50']}\")\n lines.append(\"\")\n \n # Improvement roadmap\n if report.improvement_roadmap:\n lines.append(\"IMPROVEMENT ROADMAP:\")\n for i, item in enumerate(report.improvement_roadmap[:5], 1):\n priority_symbol = \"🔴\" if item[\"priority\"] == \"HIGH\" else \"🟡\" if item[\"priority\"] == \"MEDIUM\" else \"🟢\"\n lines.append(f\" {i}. {priority_symbol} [{item['dimension']}] {item['suggestion']}\")\n lines.append(\"\")\n \n return \"\\n\".join(lines)\n\n\ndef main():\n \"\"\"Main entry point\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Score skill quality across multiple dimensions\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n python quality_scorer.py engineering/my-skill\n python quality_scorer.py engineering/my-skill --detailed --json\n python quality_scorer.py engineering/my-skill --minimum-score 75\n python quality_scorer.py engineering/my-skill --include-security\n\nQuality Dimensions (default: 4 dimensions × 25%):\n Documentation - SKILL.md quality, README, references, examples\n Code Quality - Script complexity, error handling, structure, output\n Completeness - Directory structure, assets, expected outputs, tests\n Usability - Installation simplicity, usage clarity, help accessibility\n\nWith --include-security (5 dimensions × 20%):\n Security - Sensitive data exposure, command injection, input validation\n\nLetter Grades: A+ (95+), A (90+), A- (85+), B+ (80+), B (75+), B- (70+), C+ (65+), C (60+), C- (55+), D (50+), F (\u003c50)\n \"\"\"\n )\n \n parser.add_argument(\"skill_path\",\n help=\"Path to the skill directory to assess\")\n parser.add_argument(\"--detailed\",\n action=\"store_true\",\n help=\"Show detailed component scores\")\n parser.add_argument(\"--minimum-score\",\n type=float,\n default=0,\n help=\"Minimum acceptable score (exit with error if below)\")\n parser.add_argument(\"--json\",\n action=\"store_true\",\n help=\"Output results in JSON format\")\n parser.add_argument(\"--verbose\",\n action=\"store_true\",\n help=\"Enable verbose logging\")\n parser.add_argument(\"--include-security\",\n action=\"store_true\",\n help=\"Include Security dimension (switches to 5 dimensions × 20%% each)\")\n \n args = parser.parse_args()\n \n try:\n # Create scorer and assess quality\n scorer = QualityScorer(args.skill_path, args.detailed, args.verbose, args.include_security)\n report = scorer.assess_quality()\n \n # Format and output report\n if args.json:\n print(QualityReportFormatter.format_json(report))\n else:\n print(QualityReportFormatter.format_human_readable(report, args.detailed))\n \n # Check minimum score requirement\n if report.overall_score \u003c args.minimum_score:\n print(f\"\\nERROR: Quality score {report.overall_score:.1f} is below minimum {args.minimum_score}\", file=sys.stderr)\n sys.exit(1)\n \n # Exit with different codes based on grade\n if report.letter_grade in [\"A+\", \"A\", \"A-\"]:\n sys.exit(0) # Excellent\n elif report.letter_grade in [\"B+\", \"B\", \"B-\"]:\n sys.exit(0) # Good\n elif report.letter_grade in [\"C+\", \"C\", \"C-\"]:\n sys.exit(0) # Acceptable\n elif report.letter_grade == \"D\":\n sys.exit(2) # Needs improvement\n else: # F\n sys.exit(1) # Poor quality\n \n except KeyboardInterrupt:\n print(\"\\nQuality assessment interrupted by user\", file=sys.stderr)\n sys.exit(130)\n except Exception as e:\n print(f\"Quality assessment failed: {str(e)}\", file=sys.stderr)\n if args.verbose:\n import traceback\n traceback.print_exc()\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()","content_type":"text/x-python; charset=utf-8","language":"python","size":50471,"content_sha256":"57c279b3942e7489aa81d76c6c9473783e5ebea870a5019c6f283c501f86dd5c"},{"filename":"scripts/script_tester.py","content":"#!/usr/bin/env python3\n\"\"\"\nScript Tester - Tests Python scripts in a skill directory\n\nThis script validates and tests Python scripts within a skill directory by checking\nsyntax, imports, runtime execution, argparse functionality, and output formats.\nIt ensures scripts meet quality standards and function correctly.\n\nUsage:\n python script_tester.py \u003cskill_path> [--timeout SECONDS] [--json] [--verbose]\n\nAuthor: Claude Skills Engineering Team\nVersion: 1.0.0\nDependencies: Python Standard Library Only\n\"\"\"\n\nimport argparse\nimport ast\nimport json\nimport os\nimport subprocess\nimport sys\nimport tempfile\nimport time\nfrom datetime import datetime, timezone\nfrom pathlib import Path\nfrom typing import Dict, List, Any, Optional, Tuple, Union\nimport threading\n\n\nclass TestError(Exception):\n \"\"\"Custom exception for testing errors\"\"\"\n pass\n\n\nclass ScriptTestResult:\n \"\"\"Container for individual script test results\"\"\"\n \n def __init__(self, script_path: str):\n self.script_path = script_path\n self.script_name = Path(script_path).name\n self.timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + \"Z\"\n self.tests = {}\n self.overall_status = \"PENDING\"\n self.execution_time = 0.0\n self.errors = []\n self.warnings = []\n \n def add_test(self, test_name: str, passed: bool, message: str = \"\", details: Dict = None):\n \"\"\"Add a test result\"\"\"\n self.tests[test_name] = {\n \"passed\": passed,\n \"message\": message,\n \"details\": details or {}\n }\n \n def add_error(self, error: str):\n \"\"\"Add an error message\"\"\"\n self.errors.append(error)\n \n def add_warning(self, warning: str):\n \"\"\"Add a warning message\"\"\"\n self.warnings.append(warning)\n \n def calculate_status(self):\n \"\"\"Calculate overall test status\"\"\"\n if not self.tests:\n self.overall_status = \"NO_TESTS\"\n return\n \n failed_tests = [name for name, result in self.tests.items() if not result[\"passed\"]]\n \n if not failed_tests:\n self.overall_status = \"PASS\"\n elif len(failed_tests) \u003c= len(self.tests) // 2:\n self.overall_status = \"PARTIAL\"\n else:\n self.overall_status = \"FAIL\"\n\n\nclass TestSuite:\n \"\"\"Container for all test results\"\"\"\n \n def __init__(self, skill_path: str):\n self.skill_path = skill_path\n self.timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + \"Z\"\n self.script_results = {}\n self.summary = {}\n self.global_errors = []\n \n def add_script_result(self, result: ScriptTestResult):\n \"\"\"Add a script test result\"\"\"\n self.script_results[result.script_name] = result\n \n def add_global_error(self, error: str):\n \"\"\"Add a global error message\"\"\"\n self.global_errors.append(error)\n \n def calculate_summary(self):\n \"\"\"Calculate summary statistics\"\"\"\n if not self.script_results:\n self.summary = {\n \"total_scripts\": 0,\n \"passed\": 0,\n \"partial\": 0,\n \"failed\": 0,\n \"overall_status\": \"NO_SCRIPTS\"\n }\n return\n \n statuses = [result.overall_status for result in self.script_results.values()]\n \n self.summary = {\n \"total_scripts\": len(self.script_results),\n \"passed\": statuses.count(\"PASS\"),\n \"partial\": statuses.count(\"PARTIAL\"),\n \"failed\": statuses.count(\"FAIL\"),\n \"no_tests\": statuses.count(\"NO_TESTS\")\n }\n \n # Determine overall status\n if self.summary[\"failed\"] == 0 and self.summary[\"no_tests\"] == 0:\n self.summary[\"overall_status\"] = \"PASS\"\n elif self.summary[\"passed\"] > 0:\n self.summary[\"overall_status\"] = \"PARTIAL\"\n else:\n self.summary[\"overall_status\"] = \"FAIL\"\n\n\nclass ScriptTester:\n \"\"\"Main script testing engine\"\"\"\n \n def __init__(self, skill_path: str, timeout: int = 30, verbose: bool = False):\n self.skill_path = Path(skill_path).resolve()\n self.timeout = timeout\n self.verbose = verbose\n self.test_suite = TestSuite(str(self.skill_path))\n \n def log_verbose(self, message: str):\n \"\"\"Log verbose message if verbose mode enabled\"\"\"\n if self.verbose:\n print(f\"[VERBOSE] {message}\", file=sys.stderr)\n \n def test_all_scripts(self) -> TestSuite:\n \"\"\"Main entry point - test all scripts in the skill\"\"\"\n try:\n self.log_verbose(f\"Starting script testing for {self.skill_path}\")\n \n # Check if skill path exists\n if not self.skill_path.exists():\n self.test_suite.add_global_error(f\"Skill path does not exist: {self.skill_path}\")\n return self.test_suite\n \n scripts_dir = self.skill_path / \"scripts\"\n if not scripts_dir.exists():\n self.test_suite.add_global_error(\"No scripts directory found\")\n return self.test_suite\n \n # Find all Python scripts\n python_files = list(scripts_dir.glob(\"*.py\"))\n if not python_files:\n self.test_suite.add_global_error(\"No Python scripts found in scripts directory\")\n return self.test_suite\n \n self.log_verbose(f\"Found {len(python_files)} Python scripts to test\")\n \n # Test each script\n for script_path in python_files:\n try:\n result = self.test_single_script(script_path)\n self.test_suite.add_script_result(result)\n except Exception as e:\n # Create a failed result for the script\n result = ScriptTestResult(str(script_path))\n result.add_error(f\"Failed to test script: {str(e)}\")\n result.overall_status = \"FAIL\"\n self.test_suite.add_script_result(result)\n \n # Calculate summary\n self.test_suite.calculate_summary()\n \n except Exception as e:\n self.test_suite.add_global_error(f\"Testing failed with exception: {str(e)}\")\n \n return self.test_suite\n \n def test_single_script(self, script_path: Path) -> ScriptTestResult:\n \"\"\"Test a single Python script comprehensively\"\"\"\n result = ScriptTestResult(str(script_path))\n start_time = time.time()\n \n try:\n self.log_verbose(f\"Testing script: {script_path.name}\")\n \n # Read script content\n try:\n content = script_path.read_text(encoding='utf-8')\n except Exception as e:\n result.add_test(\"file_readable\", False, f\"Cannot read file: {str(e)}\")\n result.add_error(f\"Cannot read script file: {str(e)}\")\n result.overall_status = \"FAIL\"\n return result\n \n result.add_test(\"file_readable\", True, \"Script file is readable\")\n \n # Test 1: Syntax validation\n self._test_syntax(content, result)\n \n # Test 2: Import validation \n self._test_imports(content, result)\n \n # Test 3: Argparse validation\n self._test_argparse_implementation(content, result)\n \n # Test 4: Main guard validation\n self._test_main_guard(content, result)\n \n # Test 5: Runtime execution tests\n if result.tests.get(\"syntax_valid\", {}).get(\"passed\", False):\n self._test_script_execution(script_path, result)\n \n # Test 6: Help functionality\n if result.tests.get(\"syntax_valid\", {}).get(\"passed\", False):\n self._test_help_functionality(script_path, result)\n \n # Test 7: Sample data processing (if available)\n self._test_sample_data_processing(script_path, result)\n \n # Test 8: Output format validation\n self._test_output_formats(script_path, result)\n \n except Exception as e:\n result.add_error(f\"Unexpected error during testing: {str(e)}\")\n \n finally:\n result.execution_time = time.time() - start_time\n result.calculate_status()\n \n return result\n \n def _test_syntax(self, content: str, result: ScriptTestResult):\n \"\"\"Test Python syntax validity\"\"\"\n self.log_verbose(\"Testing syntax...\")\n \n try:\n ast.parse(content)\n result.add_test(\"syntax_valid\", True, \"Python syntax is valid\")\n except SyntaxError as e:\n result.add_test(\"syntax_valid\", False, f\"Syntax error: {str(e)}\", \n {\"error\": str(e), \"line\": getattr(e, 'lineno', 'unknown')})\n result.add_error(f\"Syntax error: {str(e)}\")\n \n def _test_imports(self, content: str, result: ScriptTestResult):\n \"\"\"Test import statements for external dependencies\"\"\"\n self.log_verbose(\"Testing imports...\")\n \n try:\n tree = ast.parse(content)\n external_imports = self._find_external_imports(tree)\n \n if not external_imports:\n result.add_test(\"imports_valid\", True, \"Uses only standard library imports\")\n else:\n result.add_test(\"imports_valid\", False, \n f\"Uses external imports: {', '.join(external_imports)}\",\n {\"external_imports\": external_imports})\n result.add_error(f\"External imports detected: {', '.join(external_imports)}\")\n \n except Exception as e:\n result.add_test(\"imports_valid\", False, f\"Error analyzing imports: {str(e)}\")\n \n def _find_external_imports(self, tree: ast.AST) -> List[str]:\n \"\"\"Find external (non-stdlib) imports\"\"\"\n # Comprehensive standard library module list\n stdlib_modules = {\n # Built-in modules\n 'argparse', 'ast', 'json', 'os', 'sys', 'pathlib', 'datetime', 'typing',\n 'collections', 're', 'math', 'random', 'itertools', 'functools', 'operator',\n 'csv', 'sqlite3', 'urllib', 'http', 'html', 'xml', 'email', 'base64',\n 'hashlib', 'hmac', 'secrets', 'tempfile', 'shutil', 'glob', 'fnmatch',\n 'subprocess', 'threading', 'multiprocessing', 'queue', 'time', 'calendar',\n 'locale', 'gettext', 'logging', 'warnings', 'unittest', 'doctest',\n 'pickle', 'copy', 'pprint', 'reprlib', 'enum', 'dataclasses',\n 'contextlib', 'abc', 'atexit', 'traceback', 'gc', 'weakref', 'types',\n 'decimal', 'fractions', 'statistics', 'cmath', 'platform', 'errno',\n 'io', 'codecs', 'unicodedata', 'stringprep', 'textwrap', 'string',\n 'struct', 'difflib', 'heapq', 'bisect', 'array', 'uuid', 'mmap',\n 'ctypes', 'winreg', 'msvcrt', 'winsound', 'posix', 'pwd', 'grp',\n 'crypt', 'termios', 'tty', 'pty', 'fcntl', 'resource', 'nis',\n 'syslog', 'signal', 'socket', 'ssl', 'select', 'selectors',\n 'asyncio', 'asynchat', 'asyncore', 'netrc', 'xdrlib', 'plistlib',\n 'mailbox', 'mimetypes', 'encodings', 'pkgutil', 'modulefinder',\n 'runpy', 'importlib', 'imp', 'zipimport', 'zipfile', 'tarfile',\n 'gzip', 'bz2', 'lzma', 'zlib', 'binascii', 'quopri', 'uu',\n 'configparser', 'netrc', 'xdrlib', 'plistlib', 'token', 'tokenize',\n 'keyword', 'heapq', 'bisect', 'array', 'weakref', 'types',\n 'copyreg', 'shelve', 'marshal', 'dbm', 'sqlite3', 'zoneinfo'\n }\n \n external_imports = []\n \n for node in ast.walk(tree):\n if isinstance(node, ast.Import):\n for alias in node.names:\n module_name = alias.name.split('.')[0]\n if module_name not in stdlib_modules and not module_name.startswith('_'):\n external_imports.append(alias.name)\n \n elif isinstance(node, ast.ImportFrom) and node.module:\n module_name = node.module.split('.')[0]\n if module_name not in stdlib_modules and not module_name.startswith('_'):\n external_imports.append(node.module)\n \n return list(set(external_imports))\n \n def _test_argparse_implementation(self, content: str, result: ScriptTestResult):\n \"\"\"Test argparse implementation\"\"\"\n self.log_verbose(\"Testing argparse implementation...\")\n \n try:\n tree = ast.parse(content)\n \n # Check for argparse import\n has_argparse_import = False\n has_parser_creation = False\n has_parse_args = False\n \n for node in ast.walk(tree):\n if isinstance(node, (ast.Import, ast.ImportFrom)):\n if (isinstance(node, ast.Import) and \n any(alias.name == 'argparse' for alias in node.names)):\n has_argparse_import = True\n elif (isinstance(node, ast.ImportFrom) and \n node.module == 'argparse'):\n has_argparse_import = True\n \n elif isinstance(node, ast.Call):\n # Check for ArgumentParser creation\n if (isinstance(node.func, ast.Attribute) and\n isinstance(node.func.value, ast.Name) and\n node.func.value.id == 'argparse' and\n node.func.attr == 'ArgumentParser'):\n has_parser_creation = True\n \n # Check for parse_args call\n if (isinstance(node.func, ast.Attribute) and\n node.func.attr == 'parse_args'):\n has_parse_args = True\n \n argparse_score = sum([has_argparse_import, has_parser_creation, has_parse_args])\n \n if argparse_score == 3:\n result.add_test(\"argparse_implementation\", True, \"Complete argparse implementation found\")\n elif argparse_score > 0:\n result.add_test(\"argparse_implementation\", False, \n \"Partial argparse implementation\", \n {\"missing_components\": [\n comp for comp, present in [\n (\"import\", has_argparse_import),\n (\"parser_creation\", has_parser_creation),\n (\"parse_args\", has_parse_args)\n ] if not present\n ]})\n result.add_warning(\"Incomplete argparse implementation\")\n else:\n result.add_test(\"argparse_implementation\", False, \"No argparse implementation found\")\n result.add_error(\"Script should use argparse for command-line arguments\")\n \n except Exception as e:\n result.add_test(\"argparse_implementation\", False, f\"Error analyzing argparse: {str(e)}\")\n \n def _test_main_guard(self, content: str, result: ScriptTestResult):\n \"\"\"Test for if __name__ == '__main__' guard\"\"\"\n self.log_verbose(\"Testing main guard...\")\n \n has_main_guard = 'if __name__ == \"__main__\"' in content or \"if __name__ == '__main__'\" in content\n \n if has_main_guard:\n result.add_test(\"main_guard\", True, \"Has proper main guard\")\n else:\n result.add_test(\"main_guard\", False, \"Missing main guard\")\n result.add_error(\"Script should have 'if __name__ == \\\"__main__\\\"' guard\")\n \n def _test_script_execution(self, script_path: Path, result: ScriptTestResult):\n \"\"\"Test basic script execution\"\"\"\n self.log_verbose(\"Testing script execution...\")\n \n try:\n # Try to run the script with no arguments (should not crash immediately)\n process = subprocess.run(\n [sys.executable, str(script_path)],\n capture_output=True,\n text=True,\n timeout=self.timeout,\n cwd=script_path.parent\n )\n \n # Script might exit with error code if no args provided, but shouldn't crash\n if process.returncode in (0, 1, 2): # 0=success, 1=general error, 2=misuse\n result.add_test(\"basic_execution\", True, \n f\"Script runs without crashing (exit code: {process.returncode})\")\n else:\n result.add_test(\"basic_execution\", False,\n f\"Script crashed with exit code {process.returncode}\",\n {\"stdout\": process.stdout, \"stderr\": process.stderr})\n \n except subprocess.TimeoutExpired:\n result.add_test(\"basic_execution\", False, \n f\"Script execution timed out after {self.timeout} seconds\")\n result.add_error(f\"Script execution timeout ({self.timeout}s)\")\n \n except Exception as e:\n result.add_test(\"basic_execution\", False, f\"Execution error: {str(e)}\")\n result.add_error(f\"Script execution failed: {str(e)}\")\n \n def _test_help_functionality(self, script_path: Path, result: ScriptTestResult):\n \"\"\"Test --help functionality\"\"\"\n self.log_verbose(\"Testing help functionality...\")\n \n try:\n # Test --help flag\n process = subprocess.run(\n [sys.executable, str(script_path), '--help'],\n capture_output=True,\n text=True,\n timeout=self.timeout,\n cwd=script_path.parent\n )\n \n if process.returncode == 0:\n help_output = process.stdout\n \n # Check for reasonable help content\n help_indicators = ['usage:', 'positional arguments:', 'optional arguments:', \n 'options:', 'description:', 'help']\n has_help_content = any(indicator in help_output.lower() for indicator in help_indicators)\n \n if has_help_content and len(help_output.strip()) > 50:\n result.add_test(\"help_functionality\", True, \"Provides comprehensive help text\")\n else:\n result.add_test(\"help_functionality\", False, \n \"Help text is too brief or missing key sections\",\n {\"help_output\": help_output})\n result.add_warning(\"Help text could be more comprehensive\")\n \n else:\n result.add_test(\"help_functionality\", False, \n f\"Help command failed with exit code {process.returncode}\",\n {\"stderr\": process.stderr})\n result.add_error(\"--help flag does not work properly\")\n \n except subprocess.TimeoutExpired:\n result.add_test(\"help_functionality\", False, \"Help command timed out\")\n \n except Exception as e:\n result.add_test(\"help_functionality\", False, f\"Help test error: {str(e)}\")\n \n def _test_sample_data_processing(self, script_path: Path, result: ScriptTestResult):\n \"\"\"Test script against sample data if available\"\"\"\n self.log_verbose(\"Testing sample data processing...\")\n \n assets_dir = self.skill_path / \"assets\"\n if not assets_dir.exists():\n result.add_test(\"sample_data_processing\", True, \"No sample data to test (assets dir missing)\")\n return\n \n # Look for sample input files\n sample_files = list(assets_dir.rglob(\"*sample*\")) + list(assets_dir.rglob(\"*test*\"))\n sample_files = [f for f in sample_files if f.is_file() and not f.name.startswith('.')]\n \n if not sample_files:\n result.add_test(\"sample_data_processing\", True, \"No sample data files found to test\")\n return\n \n tested_files = 0\n successful_tests = 0\n \n for sample_file in sample_files[:3]: # Test up to 3 sample files\n try:\n self.log_verbose(f\"Testing with sample file: {sample_file.name}\")\n \n # Try to run script with the sample file as input\n process = subprocess.run(\n [sys.executable, str(script_path), str(sample_file)],\n capture_output=True,\n text=True,\n timeout=self.timeout,\n cwd=script_path.parent\n )\n \n tested_files += 1\n \n if process.returncode == 0:\n successful_tests += 1\n else:\n self.log_verbose(f\"Sample test failed for {sample_file.name}: {process.stderr}\")\n \n except subprocess.TimeoutExpired:\n tested_files += 1\n result.add_warning(f\"Sample data test timed out for {sample_file.name}\")\n except Exception as e:\n tested_files += 1\n self.log_verbose(f\"Sample test error for {sample_file.name}: {str(e)}\")\n \n if tested_files == 0:\n result.add_test(\"sample_data_processing\", True, \"No testable sample data found\")\n elif successful_tests == tested_files:\n result.add_test(\"sample_data_processing\", True, \n f\"Successfully processed all {tested_files} sample files\")\n elif successful_tests > 0:\n result.add_test(\"sample_data_processing\", False,\n f\"Processed {successful_tests}/{tested_files} sample files\",\n {\"success_rate\": successful_tests / tested_files})\n result.add_warning(\"Some sample data processing failed\")\n else:\n result.add_test(\"sample_data_processing\", False, \n \"Failed to process any sample data files\")\n result.add_error(\"Script cannot process sample data\")\n \n def _test_output_formats(self, script_path: Path, result: ScriptTestResult):\n \"\"\"Test output format compliance\"\"\"\n self.log_verbose(\"Testing output formats...\")\n \n # Test if script supports JSON output\n json_support = False\n human_readable_support = False\n \n try:\n # Read script content to check for output format indicators\n content = script_path.read_text(encoding='utf-8')\n \n # Look for JSON-related code\n if any(indicator in content.lower() for indicator in ['json.dump', 'json.load', '\"json\"', '--json']):\n json_support = True\n \n # Look for human-readable output indicators\n if any(indicator in content for indicator in ['print(', 'format(', 'f\"', \"f'\"]):\n human_readable_support = True\n \n # Try running with --json flag if it looks like it supports it\n if '--json' in content:\n try:\n process = subprocess.run(\n [sys.executable, str(script_path), '--json', '--help'],\n capture_output=True,\n text=True,\n timeout=10,\n cwd=script_path.parent\n )\n if process.returncode == 0:\n json_support = True\n except:\n pass\n \n # Evaluate dual output support\n if json_support and human_readable_support:\n result.add_test(\"output_formats\", True, \"Supports both JSON and human-readable output\")\n elif json_support or human_readable_support:\n format_type = \"JSON\" if json_support else \"human-readable\"\n result.add_test(\"output_formats\", False,\n f\"Supports only {format_type} output\",\n {\"json_support\": json_support, \"human_readable_support\": human_readable_support})\n result.add_warning(\"Consider adding dual output format support\")\n else:\n result.add_test(\"output_formats\", False, \"No clear output format support detected\")\n result.add_warning(\"Output format support is unclear\")\n \n except Exception as e:\n result.add_test(\"output_formats\", False, f\"Error testing output formats: {str(e)}\")\n\n\nclass TestReportFormatter:\n \"\"\"Formats test reports for output\"\"\"\n \n @staticmethod\n def format_json(test_suite: TestSuite) -> str:\n \"\"\"Format test suite as JSON\"\"\"\n return json.dumps({\n \"skill_path\": test_suite.skill_path,\n \"timestamp\": test_suite.timestamp,\n \"summary\": test_suite.summary,\n \"global_errors\": test_suite.global_errors,\n \"script_results\": {\n name: {\n \"script_path\": result.script_path,\n \"timestamp\": result.timestamp,\n \"overall_status\": result.overall_status,\n \"execution_time\": round(result.execution_time, 2),\n \"tests\": result.tests,\n \"errors\": result.errors,\n \"warnings\": result.warnings\n }\n for name, result in test_suite.script_results.items()\n }\n }, indent=2)\n \n @staticmethod\n def format_human_readable(test_suite: TestSuite) -> str:\n \"\"\"Format test suite as human-readable text\"\"\"\n lines = []\n lines.append(\"=\" * 60)\n lines.append(\"SCRIPT TESTING REPORT\")\n lines.append(\"=\" * 60)\n lines.append(f\"Skill: {test_suite.skill_path}\")\n lines.append(f\"Timestamp: {test_suite.timestamp}\")\n lines.append(\"\")\n \n # Summary\n if test_suite.summary:\n lines.append(\"SUMMARY:\")\n lines.append(f\" Total Scripts: {test_suite.summary['total_scripts']}\")\n lines.append(f\" Passed: {test_suite.summary['passed']}\")\n lines.append(f\" Partial: {test_suite.summary['partial']}\")\n lines.append(f\" Failed: {test_suite.summary['failed']}\")\n lines.append(f\" Overall Status: {test_suite.summary['overall_status']}\")\n lines.append(\"\")\n \n # Global errors\n if test_suite.global_errors:\n lines.append(\"GLOBAL ERRORS:\")\n for error in test_suite.global_errors:\n lines.append(f\" • {error}\")\n lines.append(\"\")\n \n # Individual script results\n for script_name, result in test_suite.script_results.items():\n lines.append(f\"SCRIPT: {script_name}\")\n lines.append(f\" Status: {result.overall_status}\")\n lines.append(f\" Execution Time: {result.execution_time:.2f}s\")\n lines.append(\"\")\n \n # Tests\n if result.tests:\n lines.append(\" TESTS:\")\n for test_name, test_result in result.tests.items():\n status = \"✓ PASS\" if test_result[\"passed\"] else \"✗ FAIL\"\n lines.append(f\" {status}: {test_result['message']}\")\n lines.append(\"\")\n \n # Errors\n if result.errors:\n lines.append(\" ERRORS:\")\n for error in result.errors:\n lines.append(f\" • {error}\")\n lines.append(\"\")\n \n # Warnings\n if result.warnings:\n lines.append(\" WARNINGS:\")\n for warning in result.warnings:\n lines.append(f\" • {warning}\")\n lines.append(\"\")\n \n lines.append(\"-\" * 40)\n lines.append(\"\")\n \n return \"\\n\".join(lines)\n\n\ndef main():\n \"\"\"Main entry point\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Test Python scripts in a skill directory\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n python script_tester.py engineering/my-skill\n python script_tester.py engineering/my-skill --timeout 60 --json\n python script_tester.py engineering/my-skill --verbose\n\nTest Categories:\n - Syntax validation (AST parsing)\n - Import validation (stdlib only)\n - Argparse implementation\n - Main guard presence\n - Basic execution testing\n - Help functionality\n - Sample data processing\n - Output format compliance\n \"\"\"\n )\n \n parser.add_argument(\"skill_path\",\n help=\"Path to the skill directory containing scripts to test\")\n parser.add_argument(\"--timeout\",\n type=int,\n default=30,\n help=\"Timeout for script execution tests in seconds (default: 30)\")\n parser.add_argument(\"--json\",\n action=\"store_true\",\n help=\"Output results in JSON format\")\n parser.add_argument(\"--verbose\",\n action=\"store_true\", \n help=\"Enable verbose logging\")\n \n args = parser.parse_args()\n \n try:\n # Create tester and run tests\n tester = ScriptTester(args.skill_path, args.timeout, args.verbose)\n test_suite = tester.test_all_scripts()\n \n # Format and output results\n if args.json:\n print(TestReportFormatter.format_json(test_suite))\n else:\n print(TestReportFormatter.format_human_readable(test_suite))\n \n # Exit with appropriate code\n if test_suite.global_errors:\n sys.exit(1)\n elif test_suite.summary.get(\"overall_status\") == \"FAIL\":\n sys.exit(1)\n elif test_suite.summary.get(\"overall_status\") == \"PARTIAL\":\n sys.exit(2) # Partial success\n else:\n sys.exit(0) # Success\n \n except KeyboardInterrupt:\n print(\"\\nTesting interrupted by user\", file=sys.stderr)\n sys.exit(130)\n except Exception as e:\n print(f\"Testing failed: {str(e)}\", file=sys.stderr)\n if args.verbose:\n import traceback\n traceback.print_exc()\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()","content_type":"text/x-python; charset=utf-8","language":"python","size":31241,"content_sha256":"997e6848d9da5ef2d81cfd05c9401c82b701bc9f371ffce9c1b862ad1adc1e72"},{"filename":"scripts/security_scorer.py","content":"#!/usr/bin/env python3\n\"\"\"\nSecurity Scorer - Security dimension scoring module\n\nThis module provides comprehensive security assessment for Python scripts,\nevaluating sensitive data exposure, safe file operations, command injection\nprevention, and input validation quality.\n\nAuthor: Claude Skills Engineering Team\nVersion: 2.0.0\n\"\"\"\n\nimport re\nfrom pathlib import Path\nfrom typing import Dict, List, Tuple, Optional, Any\n\n# =============================================================================\n# CONSTANTS - Scoring thresholds and weights\n# =============================================================================\n\n# Maximum score per component (25 points each, 4 components = 100 total)\nMAX_COMPONENT_SCORE: int = 25\n\n# Minimum score floor (never go below 0)\nMIN_SCORE: int = 0\n\n# Security score thresholds for tier recommendations\nSECURITY_SCORE_POWERFUL_TIER: int = 70 # Required for POWERFUL tier\nSECURITY_SCORE_STANDARD_TIER: int = 50 # Required for STANDARD tier\n\n# Scoring modifiers (magic numbers replaced with named constants)\nBASE_SCORE_SENSITIVE_DATA: int = 25 # Start with full points\nBASE_SCORE_FILE_OPS: int = 15 # Base score for file operations\nBASE_SCORE_COMMAND_INJECTION: int = 25 # Start with full points\nBASE_SCORE_INPUT_VALIDATION: int = 10 # Base score for input validation\n\n# Penalty amounts (negative scoring)\nCRITICAL_VULNERABILITY_PENALTY: int = -25 # Critical issues (hardcoded passwords, etc.)\nHIGH_SEVERITY_PENALTY: int = -10 # High severity issues\nMEDIUM_SEVERITY_PENALTY: int = -5 # Medium severity issues\nLOW_SEVERITY_PENALTY: int = -2 # Low severity issues\n\n# Bonus amounts (positive scoring)\nSAFE_PATTERN_BONUS: int = 2 # Bonus for using safe patterns\nGOOD_PRACTICE_BONUS: int = 3 # Bonus for good security practices\n\n# =============================================================================\n# PRE-COMPILED REGEX PATTERNS - Sensitive Data Detection\n# =============================================================================\n\n# Hardcoded credentials patterns (CRITICAL severity)\nPATTERN_HARDCODED_PASSWORD = re.compile(\n r'password\\s*=\\s*[\"\\'][^\"\\']{4,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_API_KEY = re.compile(\n r'api_key\\s*=\\s*[\"\\'][^\"\\']{8,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_SECRET = re.compile(\n r'secret\\s*=\\s*[\"\\'][^\"\\']{4,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_TOKEN = re.compile(\n r'token\\s*=\\s*[\"\\'][^\"\\']{8,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_PRIVATE_KEY = re.compile(\n r'private_key\\s*=\\s*[\"\\'][^\"\\']{20,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_AWS_KEY = re.compile(\n r'aws_access_key\\s*=\\s*[\"\\'][^\"\\']{16,}[\"\\']',\n re.IGNORECASE\n)\nPATTERN_HARDCODED_AWS_SECRET = re.compile(\n r'aws_secret\\s*=\\s*[\"\\'][^\"\\']{20,}[\"\\']',\n re.IGNORECASE\n)\n\n# Multi-line string patterns (CRITICAL severity)\nPATTERN_MULTILINE_STRING = re.compile(\n r'[\"\\']{3}[^\"\\']*?(?:password|api_key|secret|token|private_key)[^\"\\']*?[\"\\']{3}',\n re.IGNORECASE | re.DOTALL\n)\n\n# F-string patterns (HIGH severity)\nPATTERN_FSTRING_SENSITIVE = re.compile(\n r'f[\"\\'].*?(?:password|api_key|secret|token)\\s*=',\n re.IGNORECASE\n)\n\n# Base64 encoded secrets (MEDIUM severity)\nPATTERN_BASE64_SECRET = re.compile(\n r'(?:base64|b64encode|b64decode)\\s*\\([^)]*(?:password|api_key|secret|token)',\n re.IGNORECASE\n)\n\n# JWT tokens (HIGH severity)\nPATTERN_JWT_TOKEN = re.compile(\n r'eyJ[a-zA-Z0-9_-]*\\.eyJ[a-zA-Z0-9_-]*\\.[a-zA-Z0-9_-]*'\n)\n\n# Connection strings (HIGH severity)\nPATTERN_CONNECTION_STRING = re.compile(\n r'(?:connection_string|conn_string|database_url)\\s*=\\s*[\"\\'][^\"\\']*(?:password|pwd|passwd)[^\"\\']*[\"\\']',\n re.IGNORECASE\n)\n\n# Safe credential patterns (environment variables are OK)\nPATTERN_SAFE_ENV_VAR = re.compile(\n r'os\\.(?:getenv|environ)\\s*\\(\\s*[\"\\'][^\"\\']+[\"\\']',\n re.IGNORECASE\n)\n\n# =============================================================================\n# PRE-COMPILED REGEX PATTERNS - Path Traversal Detection\n# =============================================================================\n\n# Basic path traversal patterns\nPATTERN_PATH_TRAVERSAL_BASIC = re.compile(r'\\.\\.\\/')\nPATTERN_PATH_TRAVERSAL_WINDOWS = re.compile(r'\\.\\.\\\\')\n\n# URL encoded path traversal (MEDIUM severity)\nPATTERN_PATH_TRAVERSAL_URL_ENCODED = re.compile(\n r'%2e%2e%2f|%252e%252e%252f|\\.\\.%2f',\n re.IGNORECASE\n)\n\n# Unicode encoded path traversal (MEDIUM severity)\nPATTERN_PATH_TRAVERSAL_UNICODE = re.compile(\n r'\\\\u002e\\\\u002e|\\\\uff0e\\\\uff0e|\\u002e\\u002e\\/',\n re.IGNORECASE\n)\n\n# Null byte injection (HIGH severity)\nPATTERN_NULL_BYTE = re.compile(r'%00|\\\\x00|\\0')\n\n# Risky file operation patterns\nPATTERN_PATH_CONCAT = re.compile(\n r'open\\s*\\(\\s*[^)]*\\+',\n re.IGNORECASE\n)\nPATTERN_USER_INPUT_PATH = re.compile(\n r'\\.join\\s*\\(\\s*[^)]*input|os\\.path\\.join\\s*\\([^)]*request',\n re.IGNORECASE\n)\n\n# Safe file operation patterns\nPATTERN_SAFE_BASENAME = re.compile(r'os\\.path\\.basename', re.IGNORECASE)\nPATTERN_SAFE_PATHLIB = re.compile(r'pathlib\\.Path\\s*\\(', re.IGNORECASE)\nPATTERN_PATH_VALIDATION = re.compile(r'validate.*path', re.IGNORECASE)\nPATTERN_PATH_RESOLVE = re.compile(r'\\.resolve\\s*\\(', re.IGNORECASE)\n\n# =============================================================================\n# PRE-COMPILED REGEX PATTERNS - Command Injection Detection\n# =============================================================================\n\n# Dangerous patterns (CRITICAL severity)\nPATTERN_OS_SYSTEM = re.compile(r'os\\.system\\s*\\(')\nPATTERN_OS_POPEN = re.compile(r'os\\.popen\\s*\\(')\nPATTERN_EVAL = re.compile(r'eval\\s*\\(')\nPATTERN_EXEC = re.compile(r'exec\\s*\\(')\n\n# Subprocess with shell=True (HIGH severity)\nPATTERN_SUBPROCESS_SHELL_TRUE = re.compile(\n r'subprocess\\.(?:call|run|Popen|check_output)\\s*\\([^)]*shell\\s*=\\s*True',\n re.IGNORECASE\n)\n\n# Asyncio subprocess shell (HIGH severity)\nPATTERN_ASYNCIO_SHELL = re.compile(\n r'asyncio\\.create_subprocess_shell\\s*\$',\n re.IGNORECASE\n)\n\n# Pexpect spawn (HIGH severity)\nPATTERN_PEXPECT_SPAWN = re.compile(r'pexpect\\.spawn\\s*\\(', re.IGNORECASE)\n\n# Safe subprocess patterns\nPATTERN_SAFE_SUBPROCESS = re.compile(\n r'subprocess\\.(?:run|call|Popen)\\s*\\([^)]*shell\\s*=\\s*False',\n re.IGNORECASE\n)\nPATTERN_SHLEX_QUOTE = re.compile(r'shlex\\.quote', re.IGNORECASE)\nPATTERN_SHLEX_SPLIT = re.compile(r'shlex\\.split', re.IGNORECASE)\n\n# =============================================================================\n# PRE-COMPILED REGEX PATTERNS - Input Validation Detection\n# =============================================================================\n\n# Good validation patterns\nPATTERN_ARGPARSE = re.compile(r'argparse')\nPATTERN_TRY_EXCEPT = re.compile(r'try\\s*:[\\s\\S]*?except\\s+\\w*Error')\nPATTERN_INPUT_CHECK = re.compile(r'if\\s+not\\s+\\w+\\s*:')\nPATTERN_ISINSTANCE = re.compile(r'isinstance\\s*\\(')\nPATTERN_ISDIGIT = re.compile(r'\\.isdigit\\s*\\(\$')\nPATTERN_REGEX_VALIDATION = re.compile(r're\\.(?:match|search|fullmatch)\\s*\\(')\nPATTERN_VALIDATOR_CLASS = re.compile(r'Validator', re.IGNORECASE)\nPATTERN_VALIDATE_FUNC = re.compile(r'validate', re.IGNORECASE)\nPATTERN_SANITIZE_FUNC = re.compile(r'sanitize', re.IGNORECASE)\n\n\nclass SecurityScorer:\n \"\"\"\n Security dimension scoring engine.\n \n This class evaluates Python scripts for security vulnerabilities and best practices\n across four components:\n 1. Sensitive Data Exposure Prevention (25% of security score)\n 2. Safe File Operations (25% of security score)\n 3. Command Injection Prevention (25% of security score)\n 4. Input Validation Quality (25% of security score)\n \n Attributes:\n scripts: List of Python script paths to evaluate\n verbose: Whether to output verbose logging\n \"\"\"\n \n def __init__(self, scripts: List[Path], verbose: bool = False):\n \"\"\"\n Initialize the SecurityScorer.\n \n Args:\n scripts: List of Path objects pointing to Python scripts\n verbose: Enable verbose output for debugging\n \"\"\"\n self.scripts = scripts\n self.verbose = verbose\n self._findings: List[str] = []\n \n def _log_verbose(self, message: str) -> None:\n \"\"\"Log verbose message if verbose mode is enabled.\"\"\"\n if self.verbose:\n print(f\"[SECURITY] {message}\")\n \n def _get_script_content(self, script_path: Path) -> Optional[str]:\n \"\"\"\n Safely read script content.\n \n Args:\n script_path: Path to the Python script\n \n Returns:\n Script content as string, or None if read fails\n \"\"\"\n try:\n return script_path.read_text(encoding='utf-8')\n except Exception as e:\n self._log_verbose(f\"Failed to read {script_path}: {e}\")\n return None\n \n def _clamp_score(self, score: int) -> int:\n \"\"\"\n Clamp score to valid range [MIN_SCORE, MAX_COMPONENT_SCORE].\n \n Args:\n score: Raw score value\n \n Returns:\n Score clamped to valid range\n \"\"\"\n return max(MIN_SCORE, min(score, MAX_COMPONENT_SCORE))\n \n def _score_patterns(\n self,\n content: str,\n script_name: str,\n dangerous_patterns: List[Tuple[re.Pattern, str, int]],\n safe_patterns: List[Tuple[re.Pattern, str, int]],\n base_score: int\n ) -> Tuple[int, List[str]]:\n \"\"\"\n Generic pattern scoring method.\n \n This method evaluates a script against lists of dangerous and safe patterns,\n applying penalties for dangerous patterns found and bonuses for safe patterns.\n \n Args:\n content: Script content to analyze\n script_name: Name of the script (for findings)\n dangerous_patterns: List of (pattern, description, penalty) tuples\n safe_patterns: List of (pattern, description, bonus) tuples\n base_score: Starting score before adjustments\n \n Returns:\n Tuple of (final_score, findings_list)\n \"\"\"\n score = base_score\n findings = []\n \n # Check for dangerous patterns\n for pattern, description, penalty in dangerous_patterns:\n matches = pattern.findall(content)\n if matches:\n score += penalty # Penalty is negative\n findings.append(f\"{script_name}: {description} ({len(matches)} occurrence(s))\")\n \n # Check for safe patterns\n for pattern, description, bonus in safe_patterns:\n if pattern.search(content):\n score += bonus\n self._log_verbose(f\"Safe pattern found in {script_name}: {description}\")\n \n return self._clamp_score(score), findings\n \n def score_sensitive_data_exposure(self) -> Tuple[float, List[str]]:\n \"\"\"\n Score sensitive data exposure prevention.\n \n Evaluates scripts for:\n - Hardcoded passwords, API keys, secrets, tokens, private keys\n - Multi-line string credentials\n - F-string sensitive data\n - Base64 encoded secrets\n - JWT tokens\n - Connection strings with credentials\n \n Returns:\n Tuple of (average_score, findings_list)\n \"\"\"\n if not self.scripts:\n return float(MAX_COMPONENT_SCORE), []\n \n scores = []\n all_findings = []\n \n # Define dangerous patterns with severity-based penalties\n dangerous_patterns = [\n (PATTERN_HARDCODED_PASSWORD, 'hardcoded password', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_API_KEY, 'hardcoded API key', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_SECRET, 'hardcoded secret', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_TOKEN, 'hardcoded token', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_PRIVATE_KEY, 'hardcoded private key', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_AWS_KEY, 'hardcoded AWS key', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_HARDCODED_AWS_SECRET, 'hardcoded AWS secret', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_MULTILINE_STRING, 'multi-line string credential', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_FSTRING_SENSITIVE, 'f-string sensitive data', HIGH_SEVERITY_PENALTY),\n (PATTERN_BASE64_SECRET, 'base64 encoded secret', MEDIUM_SEVERITY_PENALTY),\n (PATTERN_JWT_TOKEN, 'JWT token in code', HIGH_SEVERITY_PENALTY),\n (PATTERN_CONNECTION_STRING, 'connection string with credentials', HIGH_SEVERITY_PENALTY),\n ]\n \n # Safe patterns get bonus points\n safe_patterns = [\n (PATTERN_SAFE_ENV_VAR, 'safe environment variable usage', SAFE_PATTERN_BONUS),\n ]\n \n for script_path in self.scripts:\n content = self._get_script_content(script_path)\n if content is None:\n continue\n \n score, findings = self._score_patterns(\n content=content,\n script_name=script_path.name,\n dangerous_patterns=dangerous_patterns,\n safe_patterns=safe_patterns,\n base_score=BASE_SCORE_SENSITIVE_DATA\n )\n \n scores.append(score)\n all_findings.extend(findings)\n \n avg_score = sum(scores) / len(scores) if scores else 0.0\n return avg_score, all_findings\n \n def score_safe_file_operations(self) -> Tuple[float, List[str]]:\n \"\"\"\n Score safe file operations.\n \n Evaluates scripts for:\n - Path traversal vulnerabilities (basic, URL-encoded, Unicode, null bytes)\n - Unsafe path construction\n - Safe patterns (pathlib, basename, validation)\n \n Returns:\n Tuple of (average_score, findings_list)\n \"\"\"\n if not self.scripts:\n return float(MAX_COMPONENT_SCORE), []\n \n scores = []\n all_findings = []\n \n # Dangerous patterns with severity-based penalties\n dangerous_patterns = [\n (PATTERN_PATH_TRAVERSAL_BASIC, 'basic path traversal', HIGH_SEVERITY_PENALTY),\n (PATTERN_PATH_TRAVERSAL_WINDOWS, 'Windows-style path traversal', HIGH_SEVERITY_PENALTY),\n (PATTERN_PATH_TRAVERSAL_URL_ENCODED, 'URL-encoded path traversal', HIGH_SEVERITY_PENALTY),\n (PATTERN_PATH_TRAVERSAL_UNICODE, 'Unicode-encoded path traversal', HIGH_SEVERITY_PENALTY),\n (PATTERN_NULL_BYTE, 'null byte injection', HIGH_SEVERITY_PENALTY),\n (PATTERN_PATH_CONCAT, 'potential path injection via concatenation', MEDIUM_SEVERITY_PENALTY),\n (PATTERN_USER_INPUT_PATH, 'user input in path construction', MEDIUM_SEVERITY_PENALTY),\n ]\n \n # Safe patterns get bonus points\n safe_patterns = [\n (PATTERN_SAFE_BASENAME, 'uses basename for safety', SAFE_PATTERN_BONUS),\n (PATTERN_SAFE_PATHLIB, 'uses pathlib', SAFE_PATTERN_BONUS),\n (PATTERN_PATH_VALIDATION, 'path validation', SAFE_PATTERN_BONUS),\n (PATTERN_PATH_RESOLVE, 'path resolution', SAFE_PATTERN_BONUS),\n ]\n \n for script_path in self.scripts:\n content = self._get_script_content(script_path)\n if content is None:\n continue\n \n score, findings = self._score_patterns(\n content=content,\n script_name=script_path.name,\n dangerous_patterns=dangerous_patterns,\n safe_patterns=safe_patterns,\n base_score=BASE_SCORE_FILE_OPS\n )\n \n scores.append(score)\n all_findings.extend(findings)\n \n avg_score = sum(scores) / len(scores) if scores else 0.0\n return avg_score, all_findings\n \n def score_command_injection_prevention(self) -> Tuple[float, List[str]]:\n # auditor:ignore-start (docstring listing what the scorer detects)\n \"\"\"\n Score command injection prevention.\n \n Evaluates scripts for:\n - os.system(), os.popen() usage\n - subprocess with shell=True\n - eval(), exec() usage\n - asyncio.create_subprocess_shell()\n - pexpect.spawn()\n - Safe patterns (shlex.quote, shell=False)\n \n Returns:\n Tuple of (average_score, findings_list)\n \"\"\"\n # auditor:ignore-end\n if not self.scripts:\n return float(MAX_COMPONENT_SCORE), []\n \n scores = []\n all_findings = []\n \n # Dangerous patterns with severity-based penalties\n dangerous_patterns = [\n (PATTERN_OS_SYSTEM, 'os.system usage - potential command injection', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_OS_POPEN, 'os.popen usage', HIGH_SEVERITY_PENALTY),\n (PATTERN_EVAL, 'eval usage - code injection risk', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_EXEC, 'exec usage - code injection risk', CRITICAL_VULNERABILITY_PENALTY),\n (PATTERN_SUBPROCESS_SHELL_TRUE, 'subprocess with shell=True', HIGH_SEVERITY_PENALTY),\n (PATTERN_ASYNCIO_SHELL, 'asyncio.create_subprocess_shell()', HIGH_SEVERITY_PENALTY),\n (PATTERN_PEXPECT_SPAWN, 'pexpect.spawn()', MEDIUM_SEVERITY_PENALTY),\n ]\n \n # Safe patterns get bonus points\n safe_patterns = [\n (PATTERN_SAFE_SUBPROCESS, 'safe subprocess usage (shell=False)', GOOD_PRACTICE_BONUS),\n (PATTERN_SHLEX_QUOTE, 'shell escaping with shlex.quote', GOOD_PRACTICE_BONUS),\n (PATTERN_SHLEX_SPLIT, 'safe argument splitting with shlex.split', GOOD_PRACTICE_BONUS),\n ]\n \n for script_path in self.scripts:\n content = self._get_script_content(script_path)\n if content is None:\n continue\n \n score, findings = self._score_patterns(\n content=content,\n script_name=script_path.name,\n dangerous_patterns=dangerous_patterns,\n safe_patterns=safe_patterns,\n base_score=BASE_SCORE_COMMAND_INJECTION\n )\n \n scores.append(score)\n all_findings.extend(findings)\n \n avg_score = sum(scores) / len(scores) if scores else 0.0\n return avg_score, all_findings\n \n def score_input_validation(self) -> Tuple[float, List[str]]:\n \"\"\"\n Score input validation quality.\n \n Evaluates scripts for:\n - argparse usage for CLI validation\n - Error handling patterns\n - Type checking (isinstance)\n - Regex validation\n - Validation/sanitization functions\n \n Returns:\n Tuple of (average_score, suggestions_list)\n \"\"\"\n if not self.scripts:\n return float(MAX_COMPONENT_SCORE), []\n \n scores = []\n suggestions = []\n \n # Good validation patterns (each gives bonus points)\n validation_patterns = [\n (PATTERN_ARGPARSE, GOOD_PRACTICE_BONUS),\n (PATTERN_TRY_EXCEPT, SAFE_PATTERN_BONUS),\n (PATTERN_INPUT_CHECK, SAFE_PATTERN_BONUS),\n (PATTERN_ISINSTANCE, SAFE_PATTERN_BONUS),\n (PATTERN_ISDIGIT, SAFE_PATTERN_BONUS),\n (PATTERN_REGEX_VALIDATION, SAFE_PATTERN_BONUS),\n (PATTERN_VALIDATOR_CLASS, GOOD_PRACTICE_BONUS),\n (PATTERN_VALIDATE_FUNC, SAFE_PATTERN_BONUS),\n (PATTERN_SANITIZE_FUNC, SAFE_PATTERN_BONUS),\n ]\n \n for script_path in self.scripts:\n content = self._get_script_content(script_path)\n if content is None:\n continue\n \n score = BASE_SCORE_INPUT_VALIDATION\n \n # Check for validation patterns\n for pattern, bonus in validation_patterns:\n if pattern.search(content):\n score += bonus\n \n scores.append(self._clamp_score(score))\n \n avg_score = sum(scores) / len(scores) if scores else 0.0\n \n if avg_score \u003c 15:\n suggestions.append(\"Add input validation with argparse, type checking, and error handling\")\n \n return avg_score, suggestions\n \n def get_overall_score(self) -> Dict[str, Any]:\n \"\"\"\n Calculate overall security score and return detailed results.\n \n Returns:\n Dictionary containing:\n - overall_score: Weighted average of all components\n - components: Individual component scores\n - findings: List of security issues found\n - suggestions: Improvement suggestions\n \"\"\"\n # Score each component\n sensitive_score, sensitive_findings = self.score_sensitive_data_exposure()\n file_ops_score, file_ops_findings = self.score_safe_file_operations()\n command_injection_score, command_findings = self.score_command_injection_prevention()\n input_validation_score, input_suggestions = self.score_input_validation()\n \n # Calculate overall score (equal weight: 25% each)\n overall_score = (\n sensitive_score * 0.25 +\n file_ops_score * 0.25 +\n command_injection_score * 0.25 +\n input_validation_score * 0.25\n )\n \n # Collect all findings\n all_findings = sensitive_findings + file_ops_findings + command_findings\n \n # Generate suggestions based on findings\n suggestions = input_suggestions.copy()\n if sensitive_findings:\n suggestions.append(\"Remove hardcoded credentials and use environment variables or secure config\")\n if file_ops_findings:\n suggestions.append(\"Validate and sanitize file paths, use pathlib for safe path handling\")\n if command_findings:\n suggestions.append(\"Avoid shell=True in subprocess, use shlex.quote for shell arguments\")\n \n # Critical vulnerability check - if any critical issues, cap the score\n critical_patterns = [\n PATTERN_HARDCODED_PASSWORD, PATTERN_HARDCODED_API_KEY,\n PATTERN_HARDCODED_PRIVATE_KEY, PATTERN_OS_SYSTEM,\n PATTERN_EVAL, PATTERN_EXEC\n ]\n \n has_critical = False\n for script_path in self.scripts:\n content = self._get_script_content(script_path)\n if content is None:\n continue\n for pattern in critical_patterns:\n if pattern.search(content):\n has_critical = True\n break\n if has_critical:\n break\n \n if has_critical:\n overall_score = min(overall_score, 30) # Cap at 30 if critical vulnerabilities exist\n \n return {\n 'overall_score': round(overall_score, 1),\n 'components': {\n 'sensitive_data_exposure': round(sensitive_score, 1),\n 'safe_file_operations': round(file_ops_score, 1),\n 'command_injection_prevention': round(command_injection_score, 1),\n 'input_validation': round(input_validation_score, 1),\n },\n 'findings': all_findings,\n 'suggestions': suggestions,\n 'has_critical_vulnerabilities': has_critical,\n }","content_type":"text/x-python; charset=utf-8","language":"python","size":23595,"content_sha256":"305c74ee50bd6c0885051d4fa59c3b3b91072ba21213d27934605dabb75b6a1b"},{"filename":"scripts/skill_validator.py","content":"#!/usr/bin/env python3\n\"\"\"\nSkill Validator - Validates skill directories against quality standards\n\nThis script validates a skill directory structure, documentation, and Python scripts\nagainst the claude-skills ecosystem standards. It checks for required files, proper\nformatting, and compliance with tier-specific requirements.\n\nUsage:\n python skill_validator.py \u003cskill_path> [--tier TIER] [--json] [--verbose]\n\nAuthor: Claude Skills Engineering Team\nVersion: 1.0.0\nDependencies: Python Standard Library Only\n\"\"\"\n\nimport argparse\nimport ast\nimport json\nimport re\nimport sys\ntry:\n import yaml\nexcept ImportError:\n # Minimal YAML subset: parse simple key: value frontmatter without pyyaml\n class _YamlStub:\n class YAMLError(Exception):\n pass\n @staticmethod\n def safe_load(text):\n result = {}\n for line in text.strip().splitlines():\n if ':' in line:\n key, _, value = line.partition(':')\n result[key.strip()] = value.strip()\n return result if result else None\n yaml = _YamlStub()\nimport datetime as dt\nfrom pathlib import Path\nfrom typing import Dict, List, Any, Optional, Tuple\n\n\nclass ValidationError(Exception):\n \"\"\"Custom exception for validation errors\"\"\"\n pass\n\n\nclass ValidationReport:\n \"\"\"Container for validation results\"\"\"\n \n def __init__(self, skill_path: str):\n self.skill_path = skill_path\n self.timestamp = dt.datetime.now(dt.timezone.utc).isoformat().replace(\"+00:00\", \"Z\")\n self.checks = {}\n self.warnings = []\n self.errors = []\n self.suggestions = []\n self.overall_score = 0.0\n self.compliance_level = \"FAIL\"\n \n def add_check(self, check_name: str, passed: bool, message: str = \"\", score: float = 0.0):\n \"\"\"Add a validation check result\"\"\"\n self.checks[check_name] = {\n \"passed\": passed,\n \"message\": message,\n \"score\": score\n }\n \n def add_warning(self, message: str):\n \"\"\"Add a warning message\"\"\"\n self.warnings.append(message)\n \n def add_error(self, message: str):\n \"\"\"Add an error message\"\"\"\n self.errors.append(message)\n \n def add_suggestion(self, message: str):\n \"\"\"Add an improvement suggestion\"\"\"\n self.suggestions.append(message)\n \n def calculate_overall_score(self):\n \"\"\"Calculate overall compliance score\"\"\"\n if not self.checks:\n self.overall_score = 0.0\n return\n \n total_score = sum(check[\"score\"] for check in self.checks.values())\n max_score = len(self.checks) * 1.0\n self.overall_score = (total_score / max_score) * 100 if max_score > 0 else 0.0\n \n # Determine compliance level\n if self.overall_score >= 90:\n self.compliance_level = \"EXCELLENT\"\n elif self.overall_score >= 75:\n self.compliance_level = \"GOOD\"\n elif self.overall_score >= 60:\n self.compliance_level = \"ACCEPTABLE\"\n elif self.overall_score >= 40:\n self.compliance_level = \"NEEDS_IMPROVEMENT\"\n else:\n self.compliance_level = \"POOR\"\n\n\nclass SkillValidator:\n \"\"\"Main skill validation engine\"\"\"\n \n # Tier requirements\n TIER_REQUIREMENTS = {\n \"BASIC\": {\n \"min_skill_md_lines\": 100,\n \"min_scripts\": 1,\n \"script_size_range\": (100, 300),\n \"required_dirs\": [\"scripts\"],\n \"optional_dirs\": [\"assets\", \"references\", \"expected_outputs\"],\n \"features_required\": [\"argparse\", \"main_guard\"]\n },\n \"STANDARD\": {\n \"min_skill_md_lines\": 200,\n \"min_scripts\": 1,\n \"script_size_range\": (300, 500),\n \"required_dirs\": [\"scripts\", \"assets\", \"references\"],\n \"optional_dirs\": [\"expected_outputs\"],\n \"features_required\": [\"argparse\", \"main_guard\", \"json_output\", \"help_text\"]\n },\n \"POWERFUL\": {\n \"min_skill_md_lines\": 300,\n \"min_scripts\": 2,\n \"script_size_range\": (500, 800),\n \"required_dirs\": [\"scripts\", \"assets\", \"references\", \"expected_outputs\"],\n \"optional_dirs\": [],\n \"features_required\": [\"argparse\", \"main_guard\", \"json_output\", \"help_text\", \"error_handling\"]\n }\n }\n \n # Sections the repo actually uses. Measured across all 361 real SKILL.md\n # files: no single heading appears in even 30% of them, so requiring a\n # fixed list is not defensible. These are scored as a recommendation\n # (RECOMMENDED_SECTIONS_THRESHOLD of them present earns full marks) and\n # never raise an error.\n RECOMMENDED_SKILL_MD_SECTIONS = [\n \"References\", \"Quick Start\", \"Related Skills\", \"Workflow\", \"Workflows\",\n \"When to Use\", \"Proactive Triggers\", \"Output Artifacts\",\n \"Anti-Patterns\", \"Output Format\", \"Overview\", \"Core Capabilities\",\n ]\n RECOMMENDED_SECTIONS_THRESHOLD = 2\n\n # What Claude Code actually reads from SKILL.md frontmatter. `description`\n # is what the skill listing matches against, so a skill without one is\n # invisible to model invocation.\n #\n # This list previously read [\"Name\", \"Tier\", \"Category\", \"Dependencies\",\n # \"Author\", \"Version\"] — the bold key/value convention used by the\n # assets/sample-skill fixture, not YAML frontmatter and not a schema any\n # real skill has ever followed. Every one of the 362 skills failed both\n # checks identically, so the output was noise nobody acted on.\n FRONTMATTER_REQUIRED_FIELDS = [\"name\", \"description\"]\n\n # Present on many skills and harmless, but no runtime reads them.\n FRONTMATTER_OPTIONAL_FIELDS = [\n \"when_to_use\", \"argument-hint\", \"arguments\", \"allowed-tools\",\n \"disallowed-tools\", \"disable-model-invocation\", \"user-invocable\",\n \"model\", \"effort\", \"context\", \"agent\", \"background\", \"hooks\",\n \"paths\", \"shell\",\n ]\n\n def __init__(self, skill_path: str, target_tier: Optional[str] = None, verbose: bool = False):\n self.skill_path = Path(skill_path).resolve()\n self.target_tier = target_tier\n self.verbose = verbose\n self.report = ValidationReport(str(self.skill_path))\n \n def log_verbose(self, message: str):\n \"\"\"Log verbose message if verbose mode enabled\"\"\"\n if self.verbose:\n print(f\"[VERBOSE] {message}\", file=sys.stderr)\n \n def validate_skill_structure(self) -> ValidationReport:\n \"\"\"Main validation entry point\"\"\"\n try:\n self.log_verbose(f\"Starting validation of {self.skill_path}\")\n \n # Check if path exists\n if not self.skill_path.exists():\n self.report.add_error(f\"Skill path does not exist: {self.skill_path}\")\n return self.report\n \n if not self.skill_path.is_dir():\n self.report.add_error(f\"Skill path is not a directory: {self.skill_path}\")\n return self.report\n \n # Run all validation checks\n self._validate_required_files()\n self._validate_skill_md()\n self._validate_readme()\n self._validate_directory_structure()\n self._validate_python_scripts()\n self._validate_tier_compliance()\n \n # Calculate overall score\n self.report.calculate_overall_score()\n \n self.log_verbose(f\"Validation completed. Score: {self.report.overall_score:.1f}\")\n \n except Exception as e:\n self.report.add_error(f\"Validation failed with exception: {str(e)}\")\n \n return self.report\n \n def _validate_required_files(self):\n \"\"\"Validate presence of required files\"\"\"\n self.log_verbose(\"Checking required files...\")\n \n # Check SKILL.md\n skill_md_path = self.skill_path / \"SKILL.md\"\n if skill_md_path.exists():\n self.report.add_check(\"skill_md_exists\", True, \"SKILL.md found\", 1.0)\n else:\n self.report.add_check(\"skill_md_exists\", False, \"SKILL.md missing\", 0.0)\n self.report.add_error(\"SKILL.md is required but missing\")\n \n # Check README.md\n readme_path = self.skill_path / \"README.md\"\n if readme_path.exists():\n self.report.add_check(\"readme_exists\", True, \"README.md found\", 1.0)\n else:\n self.report.add_check(\"readme_exists\", False, \"README.md missing\", 0.0)\n self.report.add_warning(\"README.md is recommended but missing\")\n self.report.add_suggestion(\"Add README.md with usage instructions and examples\")\n \n def _validate_skill_md(self):\n \"\"\"Validate SKILL.md content and format\"\"\"\n self.log_verbose(\"Validating SKILL.md...\")\n \n skill_md_path = self.skill_path / \"SKILL.md\"\n if not skill_md_path.exists():\n return\n \n try:\n content = skill_md_path.read_text(encoding='utf-8')\n lines = content.split('\\n')\n line_count = len([line for line in lines if line.strip()])\n \n # Check line count\n min_lines = self._get_tier_requirement(\"min_skill_md_lines\", 100)\n if line_count >= min_lines:\n self.report.add_check(\"skill_md_length\", True, \n f\"SKILL.md has {line_count} lines (≥{min_lines})\", 1.0)\n else:\n self.report.add_check(\"skill_md_length\", False,\n f\"SKILL.md has {line_count} lines (\u003c{min_lines})\", 0.0)\n self.report.add_error(f\"SKILL.md too short: {line_count} lines, minimum {min_lines}\")\n \n # Validate frontmatter\n self._validate_frontmatter(content)\n \n # Check required sections\n self._validate_required_sections(content)\n \n except Exception as e:\n self.report.add_check(\"skill_md_readable\", False, f\"Error reading SKILL.md: {str(e)}\", 0.0)\n self.report.add_error(f\"Cannot read SKILL.md: {str(e)}\")\n \n def _validate_frontmatter(self, content: str):\n \"\"\"Validate SKILL.md frontmatter\"\"\"\n self.log_verbose(\"Validating frontmatter...\")\n \n # Extract frontmatter\n if content.startswith('---'):\n try:\n end_marker = content.find('---', 3)\n if end_marker == -1:\n self.report.add_check(\"frontmatter_format\", False, \n \"Frontmatter closing marker not found\", 0.0)\n return\n \n frontmatter_text = content[3:end_marker].strip()\n frontmatter = yaml.safe_load(frontmatter_text)\n \n if not isinstance(frontmatter, dict):\n self.report.add_check(\"frontmatter_format\", False, \n \"Frontmatter is not a valid dictionary\", 0.0)\n return\n \n # Check required fields\n missing_fields = []\n for field in self.FRONTMATTER_REQUIRED_FIELDS:\n if field not in frontmatter:\n missing_fields.append(field)\n \n if not missing_fields:\n self.report.add_check(\"frontmatter_complete\", True, \n \"All required frontmatter fields present\", 1.0)\n else:\n self.report.add_check(\"frontmatter_complete\", False,\n f\"Missing fields: {', '.join(missing_fields)}\", 0.0)\n self.report.add_error(f\"Missing frontmatter fields: {', '.join(missing_fields)}\")\n \n except yaml.YAMLError as e:\n self.report.add_check(\"frontmatter_format\", False, \n f\"Invalid YAML frontmatter: {str(e)}\", 0.0)\n self.report.add_error(f\"Invalid YAML frontmatter: {str(e)}\")\n \n else:\n self.report.add_check(\"frontmatter_exists\", False, \n \"No frontmatter found\", 0.0)\n self.report.add_error(\"SKILL.md must start with YAML frontmatter\")\n \n def _validate_required_sections(self, content: str):\n \"\"\"Score SKILL.md against the repo's common section conventions.\n\n Advisory only: the repo has no universal section schema, so a miss is\n a hint to the author, not a validation error.\n \"\"\"\n self.log_verbose(\"Checking recommended sections...\")\n\n present = []\n for section in self.RECOMMENDED_SKILL_MD_SECTIONS:\n pattern = rf'^#+\\s*{re.escape(section)}\\s*

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

\n if re.search(pattern, content, re.MULTILINE | re.IGNORECASE):\n present.append(section)\n\n threshold = self.RECOMMENDED_SECTIONS_THRESHOLD\n if len(present) >= threshold:\n self.report.add_check(\"recommended_sections\", True,\n f\"{len(present)} recommended section(s) present: \"\n f\"{', '.join(present)}\", 1.0)\n else:\n self.report.add_check(\"recommended_sections\", False,\n f\"Only {len(present)} recommended section(s) present \"\n f\"(suggest at least {threshold} of: \"\n f\"{', '.join(self.RECOMMENDED_SKILL_MD_SECTIONS)})\", 0.0)\n self.report.add_warning(\n f\"SKILL.md has {len(present)} of the repo's common sections; \"\n f\"consider adding at least {threshold}\")\n\n def _validate_readme(self):\n \"\"\"Validate README.md content\"\"\"\n self.log_verbose(\"Validating README.md...\")\n \n readme_path = self.skill_path / \"README.md\"\n if not readme_path.exists():\n return\n \n try:\n content = readme_path.read_text(encoding='utf-8')\n \n # Check minimum content length\n if len(content.strip()) >= 200:\n self.report.add_check(\"readme_substantial\", True,\n \"README.md has substantial content\", 1.0)\n else:\n self.report.add_check(\"readme_substantial\", False,\n \"README.md content is too brief\", 0.5)\n self.report.add_suggestion(\"Expand README.md with more detailed usage instructions\")\n \n except Exception as e:\n self.report.add_check(\"readme_readable\", False,\n f\"Error reading README.md: {str(e)}\", 0.0)\n \n def _validate_directory_structure(self):\n \"\"\"Validate directory structure against tier requirements\"\"\"\n self.log_verbose(\"Validating directory structure...\")\n \n required_dirs = self._get_tier_requirement(\"required_dirs\", [\"scripts\"])\n optional_dirs = self._get_tier_requirement(\"optional_dirs\", [])\n \n # Check required directories\n missing_required = []\n for dir_name in required_dirs:\n dir_path = self.skill_path / dir_name\n if dir_path.exists() and dir_path.is_dir():\n self.report.add_check(f\"dir_{dir_name}_exists\", True,\n f\"{dir_name}/ directory found\", 1.0)\n else:\n missing_required.append(dir_name)\n self.report.add_check(f\"dir_{dir_name}_exists\", False,\n f\"{dir_name}/ directory missing\", 0.0)\n \n if missing_required:\n self.report.add_error(f\"Missing required directories: {', '.join(missing_required)}\")\n \n # Check optional directories and provide suggestions\n missing_optional = []\n for dir_name in optional_dirs:\n dir_path = self.skill_path / dir_name\n if not (dir_path.exists() and dir_path.is_dir()):\n missing_optional.append(dir_name)\n \n if missing_optional:\n self.report.add_suggestion(f\"Consider adding optional directories: {', '.join(missing_optional)}\")\n \n def _validate_python_scripts(self):\n \"\"\"Validate Python scripts in the scripts directory\"\"\"\n self.log_verbose(\"Validating Python scripts...\")\n \n scripts_dir = self.skill_path / \"scripts\"\n if not scripts_dir.exists():\n return\n \n python_files = list(scripts_dir.glob(\"*.py\"))\n min_scripts = self._get_tier_requirement(\"min_scripts\", 1)\n \n # Check minimum number of scripts\n if len(python_files) >= min_scripts:\n self.report.add_check(\"min_scripts_count\", True,\n f\"Found {len(python_files)} Python scripts (≥{min_scripts})\", 1.0)\n else:\n self.report.add_check(\"min_scripts_count\", False,\n f\"Found {len(python_files)} Python scripts (\u003c{min_scripts})\", 0.0)\n self.report.add_error(f\"Insufficient scripts: {len(python_files)}, minimum {min_scripts}\")\n \n # Validate each script\n for script_path in python_files:\n self._validate_single_script(script_path)\n \n def _validate_single_script(self, script_path: Path):\n \"\"\"Validate a single Python script\"\"\"\n script_name = script_path.name\n self.log_verbose(f\"Validating script: {script_name}\")\n \n try:\n content = script_path.read_text(encoding='utf-8')\n \n # Count lines of code (excluding empty lines and comments)\n lines = content.split('\\n')\n loc = len([line for line in lines if line.strip() and not line.strip().startswith('#')])\n \n # Check script size against tier requirements\n size_range = self._get_tier_requirement(\"script_size_range\", (100, 1000))\n min_size, max_size = size_range\n \n if min_size \u003c= loc \u003c= max_size:\n self.report.add_check(f\"script_size_{script_name}\", True,\n f\"{script_name} has {loc} LOC (within {min_size}-{max_size})\", 1.0)\n else:\n self.report.add_check(f\"script_size_{script_name}\", False,\n f\"{script_name} has {loc} LOC (outside {min_size}-{max_size})\", 0.5)\n if loc \u003c min_size:\n self.report.add_suggestion(f\"Consider expanding {script_name} (currently {loc} LOC)\")\n else:\n self.report.add_suggestion(f\"Consider refactoring {script_name} (currently {loc} LOC)\")\n \n # Parse and validate Python syntax\n try:\n tree = ast.parse(content)\n self.report.add_check(f\"script_syntax_{script_name}\", True,\n f\"{script_name} has valid Python syntax\", 1.0)\n \n # Check for required features\n self._validate_script_features(tree, script_name, content)\n \n except SyntaxError as e:\n self.report.add_check(f\"script_syntax_{script_name}\", False,\n f\"{script_name} has syntax error: {str(e)}\", 0.0)\n self.report.add_error(f\"Syntax error in {script_name}: {str(e)}\")\n \n except Exception as e:\n self.report.add_check(f\"script_readable_{script_name}\", False,\n f\"Cannot read {script_name}: {str(e)}\", 0.0)\n self.report.add_error(f\"Cannot read {script_name}: {str(e)}\")\n \n def _validate_script_features(self, tree: ast.AST, script_name: str, content: str):\n \"\"\"Validate required script features\"\"\"\n required_features = self._get_tier_requirement(\"features_required\", [\"argparse\", \"main_guard\"])\n \n # Check for argparse usage\n if \"argparse\" in required_features:\n has_argparse = self._check_argparse_usage(tree)\n self.report.add_check(f\"script_argparse_{script_name}\", has_argparse,\n f\"{'Uses' if has_argparse else 'Missing'} argparse in {script_name}\", 1.0 if has_argparse else 0.0)\n if not has_argparse:\n self.report.add_error(f\"{script_name} must use argparse for command-line arguments\")\n \n # Check for main guard\n if \"main_guard\" in required_features:\n has_main_guard = 'if __name__ == \"__main__\"' in content\n self.report.add_check(f\"script_main_guard_{script_name}\", has_main_guard,\n f\"{'Has' if has_main_guard else 'Missing'} main guard in {script_name}\", 1.0 if has_main_guard else 0.0)\n if not has_main_guard:\n self.report.add_error(f\"{script_name} must have 'if __name__ == \\\"__main__\\\"' guard\")\n \n # Check for external imports (should only use stdlib)\n external_imports = self._check_external_imports(tree)\n if not external_imports:\n self.report.add_check(f\"script_imports_{script_name}\", True,\n f\"{script_name} uses only standard library\", 1.0)\n else:\n self.report.add_check(f\"script_imports_{script_name}\", False,\n f\"{script_name} uses external imports: {', '.join(external_imports)}\", 0.0)\n self.report.add_error(f\"{script_name} uses external imports: {', '.join(external_imports)}\")\n \n def _check_argparse_usage(self, tree: ast.AST) -> bool:\n \"\"\"Check if the script uses argparse\"\"\"\n for node in ast.walk(tree):\n if isinstance(node, ast.Import):\n for alias in node.names:\n if alias.name == 'argparse':\n return True\n elif isinstance(node, ast.ImportFrom):\n if node.module == 'argparse':\n return True\n return False\n \n def _check_external_imports(self, tree: ast.AST) -> List[str]:\n \"\"\"Check for external (non-stdlib) imports.\n\n Uses the interpreter's own module list rather than a hand-maintained\n set, which is what this function's original comment asked for. The old\n set omitted `__future__`, so every script using\n `from __future__ import annotations` was reported as carrying an\n external dependency.\n \"\"\"\n stdlib_modules = set(getattr(sys, \"stdlib_module_names\", ()))\n if not stdlib_modules: # Python \u003c 3.10 has no sys.stdlib_module_names\n stdlib_modules = set(sys.builtin_module_names)\n # A compiler directive, not a dependency.\n stdlib_modules.add('__future__')\n \n external_imports = []\n for node in ast.walk(tree):\n if isinstance(node, ast.Import):\n for alias in node.names:\n module_name = alias.name.split('.')[0]\n if module_name not in stdlib_modules:\n external_imports.append(alias.name)\n elif isinstance(node, ast.ImportFrom) and node.module:\n module_name = node.module.split('.')[0]\n if module_name not in stdlib_modules:\n external_imports.append(node.module)\n \n return list(set(external_imports))\n \n def _validate_tier_compliance(self):\n \"\"\"Validate overall tier compliance\"\"\"\n if not self.target_tier:\n return\n \n self.log_verbose(f\"Validating {self.target_tier} tier compliance...\")\n \n # This is a summary check - individual checks are done in other methods\n critical_checks = [\"skill_md_exists\", \"min_scripts_count\", \"skill_md_length\"]\n failed_critical = [check for check in critical_checks \n if check in self.report.checks and not self.report.checks[check][\"passed\"]]\n \n if not failed_critical:\n self.report.add_check(\"tier_compliance\", True,\n f\"Meets {self.target_tier} tier requirements\", 1.0)\n else:\n self.report.add_check(\"tier_compliance\", False,\n f\"Does not meet {self.target_tier} tier requirements\", 0.0)\n self.report.add_error(f\"Failed critical checks for {self.target_tier} tier: {', '.join(failed_critical)}\")\n \n def _get_tier_requirement(self, requirement: str, default: Any) -> Any:\n \"\"\"Get tier-specific requirement value\"\"\"\n if self.target_tier and self.target_tier in self.TIER_REQUIREMENTS:\n return self.TIER_REQUIREMENTS[self.target_tier].get(requirement, default)\n return default\n\n\nclass ReportFormatter:\n \"\"\"Formats validation reports for output\"\"\"\n \n @staticmethod\n def format_json(report: ValidationReport) -> str:\n \"\"\"Format report as JSON\"\"\"\n return json.dumps({\n \"skill_path\": report.skill_path,\n \"timestamp\": report.timestamp,\n \"overall_score\": round(report.overall_score, 1),\n \"compliance_level\": report.compliance_level,\n \"checks\": report.checks,\n \"warnings\": report.warnings,\n \"errors\": report.errors,\n \"suggestions\": report.suggestions\n }, indent=2)\n \n @staticmethod\n def format_human_readable(report: ValidationReport) -> str:\n \"\"\"Format report as human-readable text\"\"\"\n lines = []\n lines.append(\"=\" * 60)\n lines.append(\"SKILL VALIDATION REPORT\")\n lines.append(\"=\" * 60)\n lines.append(f\"Skill: {report.skill_path}\")\n lines.append(f\"Timestamp: {report.timestamp}\")\n lines.append(f\"Overall Score: {report.overall_score:.1f}/100 ({report.compliance_level})\")\n lines.append(\"\")\n \n # Group checks by category\n structure_checks = {k: v for k, v in report.checks.items() if k.startswith(('skill_md', 'readme', 'dir_'))}\n script_checks = {k: v for k, v in report.checks.items() if k.startswith('script_')}\n other_checks = {k: v for k, v in report.checks.items() if k not in structure_checks and k not in script_checks}\n \n if structure_checks:\n lines.append(\"STRUCTURE VALIDATION:\")\n for check_name, result in structure_checks.items():\n status = \"✓ PASS\" if result[\"passed\"] else \"✗ FAIL\"\n lines.append(f\" {status}: {result['message']}\")\n lines.append(\"\")\n \n if script_checks:\n lines.append(\"SCRIPT VALIDATION:\")\n for check_name, result in script_checks.items():\n status = \"✓ PASS\" if result[\"passed\"] else \"✗ FAIL\"\n lines.append(f\" {status}: {result['message']}\")\n lines.append(\"\")\n \n if other_checks:\n lines.append(\"OTHER CHECKS:\")\n for check_name, result in other_checks.items():\n status = \"✓ PASS\" if result[\"passed\"] else \"✗ FAIL\"\n lines.append(f\" {status}: {result['message']}\")\n lines.append(\"\")\n \n if report.errors:\n lines.append(\"ERRORS:\")\n for error in report.errors:\n lines.append(f\" • {error}\")\n lines.append(\"\")\n \n if report.warnings:\n lines.append(\"WARNINGS:\")\n for warning in report.warnings:\n lines.append(f\" • {warning}\")\n lines.append(\"\")\n \n if report.suggestions:\n lines.append(\"SUGGESTIONS:\")\n for suggestion in report.suggestions:\n lines.append(f\" • {suggestion}\")\n lines.append(\"\")\n \n return \"\\n\".join(lines)\n\n\ndef main():\n \"\"\"Main entry point\"\"\"\n parser = argparse.ArgumentParser(\n description=\"Validate skill directories against quality standards\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n python skill_validator.py engineering/my-skill\n python skill_validator.py engineering/my-skill --tier POWERFUL --json\n python skill_validator.py engineering/my-skill --verbose\n\nTier Options:\n BASIC - Basic skill requirements (100+ lines SKILL.md, 1+ script)\n STANDARD - Standard skill requirements (200+ lines, advanced features)\n POWERFUL - Powerful skill requirements (300+ lines, comprehensive features)\n \"\"\"\n )\n \n parser.add_argument(\"skill_path\", \n help=\"Path to the skill directory to validate\")\n parser.add_argument(\"--tier\", \n choices=[\"BASIC\", \"STANDARD\", \"POWERFUL\"],\n help=\"Target tier for validation (optional)\")\n parser.add_argument(\"--json\", \n action=\"store_true\",\n help=\"Output results in JSON format\")\n parser.add_argument(\"--verbose\", \n action=\"store_true\",\n help=\"Enable verbose logging\")\n \n args = parser.parse_args()\n \n try:\n # Create validator and run validation\n validator = SkillValidator(args.skill_path, args.tier, args.verbose)\n report = validator.validate_skill_structure()\n \n # Format and output report\n if args.json:\n print(ReportFormatter.format_json(report))\n else:\n print(ReportFormatter.format_human_readable(report))\n \n # Exit with error code if validation failed\n if report.errors or report.overall_score \u003c 60:\n sys.exit(1)\n else:\n sys.exit(0)\n \n except KeyboardInterrupt:\n print(\"\\nValidation interrupted by user\", file=sys.stderr)\n sys.exit(130)\n except Exception as e:\n print(f\"Validation failed: {str(e)}\", file=sys.stderr)\n if args.verbose:\n import traceback\n traceback.print_exc()\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n","content_type":"text/x-python; charset=utf-8","language":"python","size":30500,"content_sha256":"13cf53cbec43bcd69090a6c46a37c1dde61fddc45c49d0a1b934347f5b733bbc"},{"filename":"tests/test_security_scorer.py","content":"#!/usr/bin/env python3\n\"\"\"\nTests for security_scorer.py - Security Dimension Scoring Tests\n\nThis test module validates the security_scorer.py module's ability to:\n- Detect hardcoded sensitive data (passwords, API keys, tokens, private keys)\n- Detect path traversal vulnerabilities\n- Detect command injection risks\n- Score input validation quality\n- Handle edge cases (empty files, environment variables, etc.)\n\nRun with: python -m unittest test_security_scorer\n\"\"\"\n\nimport tempfile\nimport unittest\nfrom pathlib import Path\n\n# Add the scripts directory to the path\nimport sys\nSCRIPTS_DIR = Path(__file__).parent.parent / \"scripts\"\nsys.path.insert(0, str(SCRIPTS_DIR))\n\nfrom security_scorer import (\n SecurityScorer,\n # Constants\n MAX_COMPONENT_SCORE,\n MIN_SCORE,\n BASE_SCORE_SENSITIVE_DATA,\n BASE_SCORE_FILE_OPS,\n BASE_SCORE_COMMAND_INJECTION,\n BASE_SCORE_INPUT_VALIDATION,\n CRITICAL_VULNERABILITY_PENALTY,\n HIGH_SEVERITY_PENALTY,\n MEDIUM_SEVERITY_PENALTY,\n LOW_SEVERITY_PENALTY,\n SAFE_PATTERN_BONUS,\n GOOD_PRACTICE_BONUS,\n # Pre-compiled patterns\n PATTERN_HARDCODED_PASSWORD,\n PATTERN_HARDCODED_API_KEY,\n PATTERN_HARDCODED_TOKEN,\n PATTERN_HARDCODED_PRIVATE_KEY,\n PATTERN_OS_SYSTEM,\n PATTERN_EVAL,\n PATTERN_EXEC,\n PATTERN_SUBPROCESS_SHELL_TRUE,\n PATTERN_SHLEX_QUOTE,\n PATTERN_SAFE_ENV_VAR,\n)\n\n\nclass TestSecurityScorerConstants(unittest.TestCase):\n \"\"\"Tests for security scorer constants.\"\"\"\n \n def test_max_component_score_value(self):\n \"\"\"Test that MAX_COMPONENT_SCORE is 25.\"\"\"\n self.assertEqual(MAX_COMPONENT_SCORE, 25)\n \n def test_min_score_value(self):\n \"\"\"Test that MIN_SCORE is 0.\"\"\"\n self.assertEqual(MIN_SCORE, 0)\n \n def test_base_scores_are_reasonable(self):\n \"\"\"Test that base scores are within valid range.\"\"\"\n self.assertGreaterEqual(BASE_SCORE_SENSITIVE_DATA, MIN_SCORE)\n self.assertLessEqual(BASE_SCORE_SENSITIVE_DATA, MAX_COMPONENT_SCORE)\n self.assertGreaterEqual(BASE_SCORE_FILE_OPS, MIN_SCORE)\n self.assertLessEqual(BASE_SCORE_FILE_OPS, MAX_COMPONENT_SCORE)\n self.assertGreaterEqual(BASE_SCORE_COMMAND_INJECTION, MIN_SCORE)\n self.assertLessEqual(BASE_SCORE_COMMAND_INJECTION, MAX_COMPONENT_SCORE)\n \n def test_penalty_values_are_negative(self):\n \"\"\"Test that penalty values are negative.\"\"\"\n self.assertLess(CRITICAL_VULNERABILITY_PENALTY, 0)\n self.assertLess(HIGH_SEVERITY_PENALTY, 0)\n self.assertLess(MEDIUM_SEVERITY_PENALTY, 0)\n self.assertLess(LOW_SEVERITY_PENALTY, 0)\n \n def test_bonus_values_are_positive(self):\n \"\"\"Test that bonus values are positive.\"\"\"\n self.assertGreater(SAFE_PATTERN_BONUS, 0)\n self.assertGreater(GOOD_PRACTICE_BONUS, 0)\n \n def test_severity_ordering(self):\n \"\"\"Test that severity penalties are ordered correctly.\"\"\"\n # Critical should be most severe (most negative)\n self.assertLess(CRITICAL_VULNERABILITY_PENALTY, HIGH_SEVERITY_PENALTY)\n self.assertLess(HIGH_SEVERITY_PENALTY, MEDIUM_SEVERITY_PENALTY)\n self.assertLess(MEDIUM_SEVERITY_PENALTY, LOW_SEVERITY_PENALTY)\n\n\nclass TestPrecompiledPatterns(unittest.TestCase):\n \"\"\"Tests for pre-compiled regex patterns.\"\"\"\n \n def test_password_pattern_detects_hardcoded(self):\n \"\"\"Test that password pattern detects hardcoded passwords.\"\"\"\n code = 'password = \"my_secret_password_123\"'\n self.assertTrue(PATTERN_HARDCODED_PASSWORD.search(code))\n \n def test_password_pattern_ignores_short_values(self):\n \"\"\"Test that password pattern ignores very short values.\"\"\"\n code = 'password = \"x\"' # Too short\n self.assertFalse(PATTERN_HARDCODED_PASSWORD.search(code))\n \n def test_api_key_pattern_detects_hardcoded(self):\n \"\"\"Test that API key pattern detects hardcoded keys.\"\"\"\n code = 'api_key = \"sk-1234567890abcdef\"'\n self.assertTrue(PATTERN_HARDCODED_API_KEY.search(code))\n \n def test_token_pattern_detects_hardcoded(self):\n \"\"\"Test that token pattern detects hardcoded tokens.\"\"\"\n code = 'token = \"ghp_1234567890abcdef\"'\n self.assertTrue(PATTERN_HARDCODED_TOKEN.search(code))\n \n def test_private_key_pattern_detects_hardcoded(self):\n \"\"\"Test that private key pattern detects hardcoded keys.\"\"\"\n code = 'private_key = \"-----BEGIN RSA PRIVATE KEY-----\"'\n self.assertTrue(PATTERN_HARDCODED_PRIVATE_KEY.search(code))\n \n def test_os_system_pattern_detects(self):\n \"\"\"Test that os.system pattern is detected.\"\"\"\n code = 'os.system(\"ls -la\")' # noqa: SEC-AUDITOR (test fixture for the security scorer; never run)\n self.assertTrue(PATTERN_OS_SYSTEM.search(code))\n \n def test_eval_pattern_detects(self):\n \"\"\"Test that eval pattern is detected.\"\"\"\n code = 'result = eval(user_input)' # noqa: SEC-AUDITOR (test fixture for the security scorer; never run)\n self.assertTrue(PATTERN_EVAL.search(code))\n \n def test_exec_pattern_detects(self):\n \"\"\"Test that exec pattern is detected.\"\"\"\n code = 'exec(user_code)' # noqa: SEC-AUDITOR (test fixture for the security scorer; never run)\n self.assertTrue(PATTERN_EXEC.search(code))\n \n def test_subprocess_shell_true_pattern_detects(self):\n \"\"\"Test that subprocess shell=True pattern is detected.\"\"\"\n code = 'subprocess.run(cmd, shell=True)' # noqa: SEC-AUDITOR (test fixture for the security scorer; never run)\n self.assertTrue(PATTERN_SUBPROCESS_SHELL_TRUE.search(code))\n \n def test_shlex_quote_pattern_detects(self):\n \"\"\"Test that shlex.quote pattern is detected.\"\"\"\n code = 'safe_cmd = shlex.quote(user_input)'\n self.assertTrue(PATTERN_SHLEX_QUOTE.search(code))\n \n def test_safe_env_var_pattern_detects(self):\n \"\"\"Test that safe environment variable pattern is detected.\"\"\"\n code = 'password = os.getenv(\"DB_PASSWORD\")'\n self.assertTrue(PATTERN_SAFE_ENV_VAR.search(code))\n\n\nclass TestSecurityScorerInit(unittest.TestCase):\n \"\"\"Tests for SecurityScorer initialization.\"\"\"\n \n def test_init_with_empty_list(self):\n \"\"\"Test initialization with empty script list.\"\"\"\n scorer = SecurityScorer([])\n self.assertEqual(scorer.scripts, [])\n self.assertFalse(scorer.verbose)\n \n def test_init_with_scripts(self):\n \"\"\"Test initialization with script list.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"test.py\"\n script_path.write_text(\"# test\")\n \n scorer = SecurityScorer([script_path])\n self.assertEqual(len(scorer.scripts), 1)\n \n def test_init_with_verbose(self):\n \"\"\"Test initialization with verbose mode.\"\"\"\n scorer = SecurityScorer([], verbose=True)\n self.assertTrue(scorer.verbose)\n\n\nclass TestSensitiveDataExposure(unittest.TestCase):\n \"\"\"Tests for sensitive data exposure scoring.\"\"\"\n \n def test_no_scripts_returns_max_score(self):\n \"\"\"Test that empty script list returns max score.\"\"\"\n scorer = SecurityScorer([])\n score, findings = scorer.score_sensitive_data_exposure()\n self.assertEqual(score, float(MAX_COMPONENT_SCORE))\n self.assertEqual(findings, [])\n \n def test_clean_script_scores_high(self):\n \"\"\"Test that clean script without sensitive data scores high.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"clean.py\"\n script_path.write_text(\"\"\"\nimport os\n\ndef get_password():\n return os.getenv(\"DB_PASSWORD\")\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n self.assertGreaterEqual(score, 20)\n self.assertEqual(len(findings), 0)\n \n def test_hardcoded_password_detected(self):\n \"\"\"Test that hardcoded password is detected and penalized.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"insecure.py\"\n script_path.write_text(\"\"\"\npassword = \"super_secret_password_123\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n self.assertTrue(any('password' in f.lower() for f in findings))\n \n def test_hardcoded_api_key_detected(self):\n \"\"\"Test that hardcoded API key is detected and penalized.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"insecure.py\"\n script_path.write_text(\"\"\"\napi_key = \"sk-1234567890abcdef123456\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n # Check for 'api' or 'hardcoded' in findings (description is 'hardcoded API key')\n self.assertTrue(any('api' in f.lower() or 'hardcoded' in f.lower() for f in findings))\n \n def test_hardcoded_token_detected(self):\n \"\"\"Test that hardcoded token is detected and penalized.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"insecure.py\"\n script_path.write_text(\"\"\"\ntoken = \"ghp_1234567890abcdef\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n \n def test_hardcoded_private_key_detected(self):\n \"\"\"Test that hardcoded private key is detected and penalized.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"insecure.py\"\n script_path.write_text(\"\"\"\nprivate_key = \"-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAJCA...\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n \n def test_environment_variable_not_flagged(self):\n \"\"\"Test that environment variable usage is not flagged as sensitive.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"secure.py\"\n # auditor:ignore-start (test fixture for the security scorer; never run)\n script_path.write_text(\"\"\"\nimport os\n\ndef get_credentials():\n password = os.getenv(\"DB_PASSWORD\")\n api_key = os.environ.get(\"API_KEY\")\n return password, api_key\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n # auditor:ignore-end\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n # Should score well because using environment variables\n self.assertGreaterEqual(score, 20)\n \n def test_empty_file_handled(self):\n \"\"\"Test that empty file is handled gracefully.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"empty.py\"\n script_path.write_text(\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n # Should return max score for empty file (no sensitive data)\n self.assertGreaterEqual(score, 20)\n \n def test_jwt_token_detected(self):\n \"\"\"Test that JWT token in code is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"jwt.py\"\n script_path.write_text(\"\"\"\n# JWT token for testing\ntoken = \"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_sensitive_data_exposure()\n \n # Should be penalized for JWT token\n self.assertLess(score, MAX_COMPONENT_SCORE)\n\n\nclass TestSafeFileOperations(unittest.TestCase):\n \"\"\"Tests for safe file operations scoring.\"\"\"\n \n def test_no_scripts_returns_max_score(self):\n \"\"\"Test that empty script list returns max score.\"\"\"\n scorer = SecurityScorer([])\n score, findings = scorer.score_safe_file_operations()\n self.assertEqual(score, float(MAX_COMPONENT_SCORE))\n \n def test_safe_pathlib_usage_scores_high(self):\n \"\"\"Test that safe pathlib usage scores high.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"safe.py\"\n script_path.write_text(\"\"\"\nfrom pathlib import Path\n\ndef read_file(filename):\n path = Path(filename).resolve()\n return path.read_text()\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_safe_file_operations()\n \n self.assertGreaterEqual(score, 15)\n \n def test_path_traversal_detected(self):\n \"\"\"Test that path traversal pattern is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"risky.py\"\n script_path.write_text(\"\"\"\ndef read_file(base_path, filename):\n # Potential path traversal vulnerability\n path = base_path + \"/../../../etc/passwd\"\n return open(path).read()\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_safe_file_operations()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n \n def test_basename_usage_scores_bonus(self):\n \"\"\"Test that basename usage gets bonus points.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"safe.py\"\n script_path.write_text(\"\"\"\nimport os\n\ndef safe_filename(user_input):\n return os.path.basename(user_input)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_safe_file_operations()\n \n self.assertGreaterEqual(score, 15)\n\n\nclass TestCommandInjectionPrevention(unittest.TestCase):\n \"\"\"Tests for command injection prevention scoring.\"\"\"\n \n def test_no_scripts_returns_max_score(self):\n \"\"\"Test that empty script list returns max score.\"\"\"\n scorer = SecurityScorer([])\n score, findings = scorer.score_command_injection_prevention()\n self.assertEqual(score, float(MAX_COMPONENT_SCORE))\n \n def test_os_system_detected(self):\n \"\"\"Test that os.system usage is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"risky.py\"\n # auditor:ignore-start (test fixture for the security scorer; never run)\n script_path.write_text(\"\"\"\nimport os\n\ndef run_command(user_input):\n os.system(\"echo \" + user_input)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n # auditor:ignore-end\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n self.assertTrue(any('os.system' in f.lower() for f in findings))\n \n def test_subprocess_shell_true_detected(self):\n \"\"\"Test that subprocess with shell=True is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"risky.py\"\n # auditor:ignore-start (test fixture for the security scorer; never run)\n script_path.write_text(\"\"\"\nimport subprocess\n\ndef run_command(cmd):\n subprocess.run(cmd, shell=True)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n # auditor:ignore-end\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n # Check for 'shell' or 'subprocess' in findings\n self.assertTrue(any('shell' in f.lower() or 'subprocess' in f.lower() for f in findings))\n \n def test_eval_detected(self):\n \"\"\"Test that eval usage is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"risky.py\"\n # auditor:ignore-start (test fixture for the security scorer; never run)\n script_path.write_text(\"\"\"\ndef evaluate(user_input):\n return eval(user_input)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n # auditor:ignore-end\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n self.assertTrue(any('eval' in f.lower() for f in findings))\n \n def test_exec_detected(self):\n \"\"\"Test that exec usage is detected.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"risky.py\"\n # auditor:ignore-start (test fixture for the security scorer; never run)\n script_path.write_text(\"\"\"\ndef execute(user_code):\n exec(user_code)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n # auditor:ignore-end\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertLess(score, MAX_COMPONENT_SCORE)\n self.assertTrue(any('exec' in f.lower() for f in findings))\n \n def test_shlex_quote_gets_bonus(self):\n \"\"\"Test that shlex.quote usage gets bonus points.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"safe.py\"\n script_path.write_text(\"\"\"\nimport shlex\nimport subprocess\n\ndef run_command(user_input):\n safe_cmd = shlex.quote(user_input)\n subprocess.run([\"echo\", safe_cmd], shell=False)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertGreaterEqual(score, 20)\n \n def test_safe_subprocess_scores_high(self):\n \"\"\"Test that safe subprocess usage (shell=False) scores high.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"safe.py\"\n script_path.write_text(\"\"\"\nimport subprocess\n\ndef run_command(cmd_parts):\n subprocess.run(cmd_parts, shell=False)\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, findings = scorer.score_command_injection_prevention()\n \n self.assertGreaterEqual(score, 20)\n\n\nclass TestInputValidation(unittest.TestCase):\n \"\"\"Tests for input validation scoring.\"\"\"\n \n def test_no_scripts_returns_max_score(self):\n \"\"\"Test that empty script list returns max score.\"\"\"\n scorer = SecurityScorer([])\n score, suggestions = scorer.score_input_validation()\n self.assertEqual(score, float(MAX_COMPONENT_SCORE))\n \n def test_argparse_usage_scores_bonus(self):\n \"\"\"Test that argparse usage gets bonus points.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"good.py\"\n script_path.write_text(\"\"\"\nimport argparse\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"input\")\n args = parser.parse_args()\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, suggestions = scorer.score_input_validation()\n \n # Base score is 10, argparse gives +3 bonus, so score should be 13\n self.assertGreaterEqual(score, 10)\n self.assertLessEqual(score, MAX_COMPONENT_SCORE)\n \n def test_isinstance_usage_scores_bonus(self):\n \"\"\"Test that isinstance usage gets bonus points.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"good.py\"\n script_path.write_text(\"\"\"\ndef process(value):\n if isinstance(value, str):\n return value.upper()\n return value\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, suggestions = scorer.score_input_validation()\n \n self.assertGreater(score, BASE_SCORE_INPUT_VALIDATION)\n \n def test_try_except_scores_bonus(self):\n \"\"\"Test that try/except usage gets bonus points.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"good.py\"\n script_path.write_text(\"\"\"\ndef process(value):\n try:\n return int(value)\n except ValueError:\n return 0\n \ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, suggestions = scorer.score_input_validation()\n \n self.assertGreater(score, BASE_SCORE_INPUT_VALIDATION)\n \n def test_minimal_validation_gets_suggestion(self):\n \"\"\"Test that minimal validation triggers suggestion.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"minimal.py\"\n script_path.write_text(\"\"\"\ndef main():\n print(\"Hello\")\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n score, suggestions = scorer.score_input_validation()\n \n self.assertLess(score, 15)\n self.assertTrue(len(suggestions) > 0)\n\n\nclass TestOverallScore(unittest.TestCase):\n \"\"\"Tests for overall security score calculation.\"\"\"\n \n def test_overall_score_components_present(self):\n \"\"\"Test that overall score includes all components.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"test.py\"\n script_path.write_text(\"\"\"\nimport os\nimport argparse\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.parse_args()\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n results = scorer.get_overall_score()\n \n self.assertIn('overall_score', results)\n self.assertIn('components', results)\n self.assertIn('findings', results)\n self.assertIn('suggestions', results)\n \n components = results['components']\n self.assertIn('sensitive_data_exposure', components)\n self.assertIn('safe_file_operations', components)\n self.assertIn('command_injection_prevention', components)\n self.assertIn('input_validation', components)\n \n def test_overall_score_is_weighted_average(self):\n \"\"\"Test that overall score is calculated as weighted average.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"test.py\"\n script_path.write_text(\"\"\"\nimport argparse\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.parse_args()\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n results = scorer.get_overall_score()\n \n # Calculate expected weighted average\n expected = (\n results['components']['sensitive_data_exposure'] * 0.25 +\n results['components']['safe_file_operations'] * 0.25 +\n results['components']['command_injection_prevention'] * 0.25 +\n results['components']['input_validation'] * 0.25\n )\n \n self.assertAlmostEqual(results['overall_score'], expected, places=0)\n \n def test_critical_vulnerability_caps_score(self):\n \"\"\"Test that critical vulnerabilities cap the overall score.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"critical.py\"\n script_path.write_text(\"\"\"\npassword = \"hardcoded_password_123\"\napi_key = \"sk-1234567890abcdef\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n results = scorer.get_overall_score()\n \n # Score should be capped at 30 for critical vulnerabilities\n self.assertLessEqual(results['overall_score'], 30)\n self.assertTrue(results['has_critical_vulnerabilities'])\n \n def test_secure_script_scores_high(self):\n \"\"\"Test that secure script scores high overall.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n script_path = Path(tmpdir) / \"secure.py\"\n script_path.write_text(\"\"\"\n#!/usr/bin/env python3\nimport argparse\nimport os\nimport shlex\nimport subprocess\nfrom pathlib import Path\n\ndef validate_path(path_str):\n path = Path(path_str).resolve()\n if not path.exists():\n raise FileNotFoundError(\"Path not found\")\n return path\n\ndef safe_command(args):\n return subprocess.run(args, shell=False, capture_output=True)\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"input\")\n args = parser.parse_args()\n \n db_password = os.getenv(\"DB_PASSWORD\")\n path = validate_path(args.input)\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([script_path])\n results = scorer.get_overall_score()\n \n # Secure script should score reasonably well\n # Note: Score may vary based on pattern detection\n self.assertGreater(results['overall_score'], 20)\n\n\nclass TestScoreClamping(unittest.TestCase):\n \"\"\"Tests for score boundary clamping.\"\"\"\n \n def test_score_never_below_zero(self):\n \"\"\"Test that score never goes below 0.\"\"\"\n scorer = SecurityScorer([])\n # Test with extreme negative\n result = scorer._clamp_score(-100)\n self.assertEqual(result, MIN_SCORE)\n \n def test_score_never_above_max(self):\n \"\"\"Test that score never goes above max.\"\"\"\n scorer = SecurityScorer([])\n # Test with extreme positive\n result = scorer._clamp_score(1000)\n self.assertEqual(result, MAX_COMPONENT_SCORE)\n \n def test_score_unchanged_in_valid_range(self):\n \"\"\"Test that score is unchanged in valid range.\"\"\"\n scorer = SecurityScorer([])\n for test_score in [0, 5, 10, 15, 20, 25]:\n result = scorer._clamp_score(test_score)\n self.assertEqual(result, test_score)\n\n\nclass TestMultipleScripts(unittest.TestCase):\n \"\"\"Tests for scoring multiple scripts.\"\"\"\n \n def test_multiple_scripts_averaged(self):\n \"\"\"Test that scores are averaged across multiple scripts.\"\"\"\n with tempfile.TemporaryDirectory() as tmpdir:\n secure_script = Path(tmpdir) / \"secure.py\"\n secure_script.write_text(\"\"\"\nimport os\n\ndef main():\n password = os.getenv(\"PASSWORD\")\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n insecure_script = Path(tmpdir) / \"insecure.py\"\n insecure_script.write_text(\"\"\"\npassword = \"hardcoded_secret_password\"\n\ndef main():\n pass\n \nif __name__ == \"__main__\":\n main()\n\"\"\")\n \n scorer = SecurityScorer([secure_script, insecure_script])\n score, findings = scorer.score_sensitive_data_exposure()\n \n # Score should be between secure and insecure\n self.assertGreater(score, 0)\n self.assertLess(score, MAX_COMPONENT_SCORE)\n\n\nif __name__ == \"__main__\":\n unittest.main(verbosity=2)","content_type":"text/x-python; charset=utf-8","language":"python","size":29245,"content_sha256":"e7edd1af29e0edefdfa63fb149b7a31a3439e303eb2df80e4217ea0be53dc5fc"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Skill Tester","type":"text"}]},{"type":"paragraph","content":[{"text":"Tier","type":"text","marks":[{"type":"strong"}]},{"text":": POWERFUL · ","type":"text"},{"text":"Category","type":"text","marks":[{"type":"strong"}]},{"text":": Engineering Quality Assurance · ","type":"text"},{"text":"Dependencies","type":"text","marks":[{"type":"strong"}]},{"text":": None (Python stdlib only)","type":"text"}]},{"type":"paragraph","content":[{"text":"Meta-skill that validates, tests, and scores skills in this repository. Four tools, run from the ","type":"text"},{"text":"repo root","type":"text","marks":[{"type":"strong"}]},{"text":" with full paths:","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"scripts/skill_validator.py","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" — structure + documentation compliance","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"scripts/script_tester.py","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" — Python script syntax/imports/runtime/output testing","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"scripts/quality_scorer.py","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" — multi-dimensional scoring with letter grade","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"scripts/security_scorer.py","type":"text","marks":[{"type":"code_inline"},{"type":"strong"}]},{"text":" — security posture scoring (also available via ","type":"text"},{"text":"quality_scorer.py --include-security","type":"text","marks":[{"type":"code_inline"}]},{"text":")","type":"text"}]}]}]},{"type":"blockquote","content":[{"type":"paragraph","content":[{"text":"Scope note:","type":"text","marks":[{"type":"strong"}]},{"text":" this skill's tier line-count minimums measure ","type":"text"},{"text":"legacy","type":"text","marks":[{"type":"em"}]},{"text":" skills. For authoring ","type":"text"},{"text":"new","type":"text","marks":[{"type":"em"}]},{"text":" skills, ","type":"text"},{"text":"engineering/write-a-skill","type":"text","marks":[{"type":"code_inline"}]},{"text":" (SKILL.md under ~100 lines, Matt Pocock doctrine) is the binding standard — do not pad a new skill to satisfy a tier minimum here.","type":"text"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Quick Start (exact, runnable from repo root)","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"# 1. Validate structure (exit non-zero on failure — usable as a gate)\npython3 engineering/skills/skill-tester/scripts/skill_validator.py engineering/skills/self-eval --json\n\n# 2. Test the skill's Python scripts (30s default timeout per script)\npython3 engineering/skills/skill-tester/scripts/script_tester.py engineering/skills/self-eval --json\n\n# 3. Score quality (fail CI below threshold with --minimum-score)\npython3 engineering/skills/skill-tester/scripts/quality_scorer.py engineering/skills/self-eval --json --detailed --minimum-score 75","type":"text"}]},{"type":"paragraph","content":[{"text":"Consume the JSON: validator emits ","type":"text"},{"text":"overall_score","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"compliance_level","type":"text","marks":[{"type":"code_inline"}]},{"text":", per-check ","type":"text"},{"text":"checks{}","type":"text","marks":[{"type":"code_inline"}]},{"text":"; scorer emits ","type":"text"},{"text":"overall_score","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"letter_grade","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"tier_recommendation","type":"text","marks":[{"type":"code_inline"}]},{"text":", ","type":"text"},{"text":"dimensions","type":"text","marks":[{"type":"code_inline"}]},{"text":", and an ","type":"text"},{"text":"improvement_roadmap","type":"text","marks":[{"type":"code_inline"}]},{"text":" — work the roadmap top-down, then re-run until the target score is met.","type":"text"}]},{"type":"paragraph","content":[{"text":"For repo-wide auditing prefer ","type":"text"},{"text":"scripts/audit_skills.py","type":"text","marks":[{"type":"code_inline"}]},{"text":" at the repo root (wraps the write-a-skill checklist runner across all skills).","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"What Each Tool Checks","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"skill_validator.py","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"SKILL.md frontmatter parsing, required sections, minimum line counts per tier (","type":"text"},{"text":"--tier BASIC|STANDARD|POWERFUL","type":"text","marks":[{"type":"code_inline"}]},{"text":")","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Required structure: SKILL.md, README.md, scripts/, references/, assets/, expected_outputs/","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Python scripts: argparse present, stdlib-only imports","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"script_tester.py","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"AST-based syntax validation; import analysis (flags external dependencies)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Controlled execution with timeout protection (","type":"text"},{"text":"--timeout","type":"text","marks":[{"type":"code_inline"}]},{"text":", default 30s)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"--help","type":"text","marks":[{"type":"code_inline"}]},{"text":" functionality verification; sample-data runs compared against expected_outputs/","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"quality_scorer.py","type":"text"}]},{"type":"paragraph","content":[{"text":"Four dimensions, 25% each: ","type":"text"},{"text":"Documentation","type":"text","marks":[{"type":"strong"}]},{"text":" (depth, examples, references), ","type":"text"},{"text":"Code Quality","type":"text","marks":[{"type":"strong"}]},{"text":" (complexity, error handling, output consistency), ","type":"text"},{"text":"Completeness","type":"text","marks":[{"type":"strong"}]},{"text":" (required dirs, sample data, expected outputs), ","type":"text"},{"text":"Usability","type":"text","marks":[{"type":"strong"}]},{"text":" (help text, example clarity). Outputs 0-100 + A-F grade + tier recommendation.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Tier Classification","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Tier","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"SKILL.md","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Scripts","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"CLI surface","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"BASIC","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"≥ 100 lines","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"1 (100-300 LOC)","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"basic argparse","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"STANDARD","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"≥ 200 lines","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"1-2 (300-500 LOC)","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"subcommands, JSON + text output","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"POWERFUL","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"≥ 300 lines","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"2-3 (500-800 LOC)","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"multiple modes, CI integration","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"(Advisory for legacy skills; new skills follow write-a-skill — see scope note above.)","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"CI Integration","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"yaml"},"content":[{"text":"# GitHub Actions: gate changed skills\n- name: \"validate-changed-skills\"\n run: |\n for skill in $changed_skills; do\n python3 engineering/skills/skill-tester/scripts/skill_validator.py \"$skill\" --json\n python3 engineering/skills/skill-tester/scripts/script_tester.py \"$skill\"\n python3 engineering/skills/skill-tester/scripts/quality_scorer.py \"$skill\" --minimum-score 75\n done","type":"text"}]},{"type":"paragraph","content":[{"text":"Pre-commit hook: run the validator on the staged skill directory and block the commit on non-zero exit.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Verification Loop","type":"text"}]},{"type":"paragraph","content":[{"text":"A skill \"passes\" when, in one run from repo root:","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"skill_validator.py \u003cskill> --json","type":"text","marks":[{"type":"code_inline"}]},{"text":" exits 0,","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"script_tester.py \u003cskill>","type":"text","marks":[{"type":"code_inline"}]},{"text":" reports all scripts passing, and","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"quality_scorer.py \u003cskill> --minimum-score \u003ctarget>","type":"text","marks":[{"type":"code_inline"}]},{"text":" exits 0.","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"If any step fails, apply the top ","type":"text"},{"text":"improvement_roadmap","type":"text","marks":[{"type":"code_inline"}]},{"text":" item and re-run all three — never report a partial pass.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Troubleshooting","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Timeout errors","type":"text","marks":[{"type":"strong"}]},{"text":" → raise ","type":"text"},{"text":"--timeout","type":"text","marks":[{"type":"code_inline"}]},{"text":" or optimize the script under test","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Import failures","type":"text","marks":[{"type":"strong"}]},{"text":" → external deps detected; stdlib-only is the repo policy","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Tier misclassification","type":"text","marks":[{"type":"strong"}]},{"text":" → check line counts/LOC against the tier table; remember the write-a-skill exception for new skills","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"References: ","type":"text"},{"text":"references/","type":"text","marks":[{"type":"code_inline"}]},{"text":" holds the structure specification, tier requirements matrix, and scoring rubric the tools implement.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"More in this skill","type":"text"}]},{"type":"paragraph","content":[{"text":"Load these only when the task needs them.","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"references/quality-scoring-rubric.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/quality-scoring-rubric.md","title":null}}]},{"text":" — Quality Scoring Rubric","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"references/skill-structure-specification.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/skill-structure-specification.md","title":null}}]},{"text":" — Skill Structure Specification","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"references/tier-requirements-matrix.md","type":"text","marks":[{"type":"link","attrs":{"href":"references/tier-requirements-matrix.md","title":null}}]},{"text":" — Tier Requirements Matrix","type":"text"}]}]}]}]},"metadata":{"date":"2026-11-02","name":"skill-tester","author":"@skillopedia","source":{"stars":29164,"repo_name":"claude-skills","origin_url":"https://github.com/alirezarezvani/claude-skills/blob/HEAD/engineering/skills/skill-tester/SKILL.md","repo_owner":"alirezarezvani","body_sha256":"26781603de5aa603213b3cd452fc52928caddec86fedee3640baf1089bde94d9","cluster_key":"af0238be8323106d1af9a0a057c7fcce702645e73e6346dd68bd765259bd04bc","clean_bundle":{"format":"clean-skill-bundle-v1","source":"alirezarezvani/claude-skills/engineering/skills/skill-tester/SKILL.md","text_bytes":224243,"source_commit":"9ef0c3c213932e645cc5cbeeb3c35c3a762a2437","skills_sh_rank":17359,"attachment_count":10,"text_attachments":10,"attachment_storage":"skill_attachments","binary_attachments":0},"cluster_size":1,"skill_md_path":"engineering/skills/skill-tester/SKILL.md","import_metadata":{"date":"2026-11-02","author":"@skillopedia","version":"v1","category":"general","category_label":"General"},"exact_dupes_collapsed_into_this":0},"version":"v1","category":"general","import_tag":"skills-sh-top20k-2026-11","description":"Use when authoring or auditing a skill in this ecosystem, before release. Validates structure with skill_validator.py, runs its scripts with script_tester.py and grades quality and security with quality_scorer.py and security_scorer.py."}},"renderedAt":1793666465471}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.