data-profiler — Skillopedia

Data Profiler for Construction Overview Analyze construction data to understand its characteristics, distributions, quality, and patterns. Essential for data quality assessment, ETL planning, and identifying data issues before they impact projects. Business Case Before using any construction data, you need to understand: - What data types are present - Distribution of values - Missing data patterns - Anomalies and outliers - Referential integrity issues This skill profiles data to answer these questions and provides actionable insights. Technical Implementation Quick Start Common Use Cases 1.…

,\n 'project_id': r'^[A-Z]{2,4}[-_]?\\d{3,6}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'cost_code': r'^\\d{2}[-.]?\\d{2,4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'wbs': r'^[\\d.]+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'phone': r'^\$?\\d{3}\$?[-.\\s]?\\d{3}[-.\\s]?\\d{4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'email': r'^[\\w.-]+@[\\w.-]+\\.\\w+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'date_iso': r'^\\d{4}-\\d{2}-\\d{2}',\n 'date_us': r'^\\d{1,2}/\\d{1,2}/\\d{2,4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'currency': r'^\\$?[\\d,]+\\.?\\d{0,2}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'percentage': r'^\\d+\\.?\\d*%?

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n }\n\n # Construction-specific column name patterns\n COLUMN_TYPE_HINTS = {\n 'project': ['project_id', 'project_name', 'proj', 'job'],\n 'cost': ['cost', 'amount', 'price', 'total', 'budget', 'actual'],\n 'date': ['date', 'start', 'finish', 'end', 'created', 'modified'],\n 'quantity': ['qty', 'quantity', 'count', 'units'],\n 'csi': ['csi', 'division', 'masterformat', 'spec'],\n 'location': ['location', 'area', 'zone', 'floor', 'level'],\n 'person': ['owner', 'manager', 'superintendent', 'foreman', 'contact'],\n }\n\n def __init__(self):\n self.profiles: Dict[str, DataProfile] = {}\n\n def profile_dataframe(self, df: pd.DataFrame, source_name: str) -> DataProfile:\n \"\"\"Profile a pandas DataFrame.\"\"\"\n columns = []\n\n for col in df.columns:\n col_profile = self._profile_column(df[col], col)\n columns.append(col_profile)\n\n # Calculate duplicates\n duplicate_rows = len(df) - len(df.drop_duplicates())\n\n # Calculate memory usage\n memory_bytes = df.memory_usage(deep=True).sum()\n if memory_bytes \u003c 1024:\n memory_usage = f\"{memory_bytes} B\"\n elif memory_bytes \u003c 1024**2:\n memory_usage = f\"{memory_bytes/1024:.1f} KB\"\n else:\n memory_usage = f\"{memory_bytes/1024**2:.1f} MB\"\n\n # Calculate overall quality score\n quality_score = self._calculate_quality_score(columns)\n\n # Generate recommendations\n recommendations = self._generate_recommendations(columns, df)\n\n profile = DataProfile(\n source_name=source_name,\n row_count=len(df),\n column_count=len(df.columns),\n columns=columns,\n duplicate_rows=duplicate_rows,\n memory_usage=memory_usage,\n profiled_at=datetime.now(),\n quality_score=quality_score,\n recommendations=recommendations\n )\n\n self.profiles[source_name] = profile\n return profile\n\n def _profile_column(self, series: pd.Series, name: str) -> ColumnProfile:\n \"\"\"Profile a single column.\"\"\"\n total_count = len(series)\n null_count = series.isnull().sum()\n null_percentage = (null_count / total_count * 100) if total_count > 0 else 0\n\n # Get non-null values for analysis\n non_null = series.dropna()\n unique_count = non_null.nunique()\n uniqueness_ratio = unique_count / len(non_null) if len(non_null) > 0 else 0\n\n profile = ColumnProfile(\n name=name,\n data_type=str(series.dtype),\n inferred_type=self._infer_construction_type(series, name),\n total_count=total_count,\n null_count=null_count,\n null_percentage=round(null_percentage, 2),\n unique_count=unique_count,\n uniqueness_ratio=round(uniqueness_ratio, 4)\n )\n\n # Numeric analysis\n if pd.api.types.is_numeric_dtype(series):\n profile.min_value = float(non_null.min()) if len(non_null) > 0 else None\n profile.max_value = float(non_null.max()) if len(non_null) > 0 else None\n profile.mean_value = float(non_null.mean()) if len(non_null) > 0 else None\n profile.median_value = float(non_null.median()) if len(non_null) > 0 else None\n profile.std_dev = float(non_null.std()) if len(non_null) > 1 else None\n\n # Check for outliers\n if len(non_null) > 10 and profile.std_dev:\n outliers = non_null[abs(non_null - profile.mean_value) > 3 * profile.std_dev]\n if len(outliers) > 0:\n profile.quality_issues.append(f\"{len(outliers)} potential outliers detected\")\n\n # Check for negative costs\n if any(hint in name.lower() for hint in ['cost', 'amount', 'price', 'total']):\n negatives = (non_null \u003c 0).sum()\n if negatives > 0:\n profile.quality_issues.append(f\"{negatives} negative values in cost column\")\n\n # String analysis\n elif pd.api.types.is_object_dtype(series) or pd.api.types.is_string_dtype(series):\n str_series = non_null.astype(str)\n lengths = str_series.str.len()\n profile.min_length = int(lengths.min()) if len(lengths) > 0 else None\n profile.max_length = int(lengths.max()) if len(lengths) > 0 else None\n profile.avg_length = float(lengths.mean()) if len(lengths) > 0 else None\n\n # Detect patterns\n profile.common_patterns = self._detect_patterns(str_series)\n\n # Top values\n if len(non_null) > 0:\n value_counts = non_null.value_counts().head(5)\n profile.top_values = list(zip(value_counts.index.tolist(), value_counts.values.tolist()))\n\n # Quality checks\n if null_percentage > 50:\n profile.quality_issues.append(\"High null rate (>50%)\")\n if uniqueness_ratio == 1.0 and total_count > 100:\n profile.quality_issues.append(\"All unique values - possible ID column\")\n if uniqueness_ratio \u003c 0.01 and unique_count > 1:\n profile.quality_issues.append(\"Low cardinality - possible category\")\n\n return profile\n\n def _infer_construction_type(self, series: pd.Series, name: str) -> str:\n \"\"\"Infer construction-specific data type.\"\"\"\n name_lower = name.lower()\n\n # Check column name hints\n for type_name, hints in self.COLUMN_TYPE_HINTS.items():\n if any(hint in name_lower for hint in hints):\n return type_name\n\n # Check data patterns\n non_null = series.dropna().astype(str)\n if len(non_null) == 0:\n return \"unknown\"\n\n sample = non_null.head(100)\n\n for pattern_name, pattern in self.CONSTRUCTION_PATTERNS.items():\n matches = sample.str.match(pattern, na=False).sum()\n if matches / len(sample) > 0.8:\n return pattern_name\n\n # Default to pandas dtype\n if pd.api.types.is_numeric_dtype(series):\n return \"numeric\"\n elif pd.api.types.is_datetime64_any_dtype(series):\n return \"datetime\"\n else:\n return \"text\"\n\n def _detect_patterns(self, str_series: pd.Series) -> List[str]:\n \"\"\"Detect common patterns in string data.\"\"\"\n patterns_found = []\n\n sample = str_series.head(1000)\n\n for pattern_name, pattern in self.CONSTRUCTION_PATTERNS.items():\n matches = sample.str.match(pattern, na=False).sum()\n if matches / len(sample) > 0.1:\n patterns_found.append(f\"{pattern_name} ({matches/len(sample):.0%})\")\n\n return patterns_found[:3]\n\n def _calculate_quality_score(self, columns: List[ColumnProfile]) -> float:\n \"\"\"Calculate overall data quality score (0-100).\"\"\"\n if not columns:\n return 0.0\n\n scores = []\n\n for col in columns:\n col_score = 100\n\n # Penalize for nulls\n col_score -= min(col.null_percentage, 50)\n\n # Penalize for quality issues\n col_score -= len(col.quality_issues) * 10\n\n scores.append(max(col_score, 0))\n\n return round(sum(scores) / len(scores), 1)\n\n def _generate_recommendations(self, columns: List[ColumnProfile], df: pd.DataFrame) -> List[str]:\n \"\"\"Generate recommendations based on profile.\"\"\"\n recommendations = []\n\n # High null columns\n high_null = [c for c in columns if c.null_percentage > 30]\n if high_null:\n recommendations.append(\n f\"Review {len(high_null)} columns with >30% null values: \"\n f\"{', '.join(c.name for c in high_null[:3])}\"\n )\n\n # Potential ID columns without uniqueness\n for col in columns:\n if 'id' in col.name.lower() and col.uniqueness_ratio \u003c 1.0:\n recommendations.append(\n f\"Column '{col.name}' appears to be an ID but has duplicate values\"\n )\n\n # Date columns that should be datetime\n for col in columns:\n if col.inferred_type in ['date_iso', 'date_us'] and col.data_type == 'object':\n recommendations.append(\n f\"Convert '{col.name}' to datetime type for better analysis\"\n )\n\n # Cost columns that are strings\n for col in columns:\n if col.inferred_type == 'currency' and col.data_type == 'object':\n recommendations.append(\n f\"Convert '{col.name}' to numeric type (remove $ and commas)\"\n )\n\n return recommendations\n\n def profile_to_dict(self, profile: DataProfile) -> Dict:\n \"\"\"Convert profile to dictionary for JSON export.\"\"\"\n return {\n 'source_name': profile.source_name,\n 'row_count': profile.row_count,\n 'column_count': profile.column_count,\n 'duplicate_rows': profile.duplicate_rows,\n 'memory_usage': profile.memory_usage,\n 'profiled_at': profile.profiled_at.isoformat(),\n 'quality_score': profile.quality_score,\n 'recommendations': profile.recommendations,\n 'columns': [\n {\n 'name': c.name,\n 'data_type': c.data_type,\n 'inferred_type': c.inferred_type,\n 'null_percentage': c.null_percentage,\n 'unique_count': c.unique_count,\n 'quality_issues': c.quality_issues,\n 'top_values': c.top_values[:3]\n }\n for c in profile.columns\n ]\n }\n\n def generate_profile_report(self, profile: DataProfile) -> str:\n \"\"\"Generate markdown profile report.\"\"\"\n report = [f\"# Data Profile: {profile.source_name}\", \"\"]\n report.append(f\"**Profiled At:** {profile.profiled_at.strftime('%Y-%m-%d %H:%M')}\")\n report.append(f\"**Quality Score:** {profile.quality_score}/100\")\n report.append(\"\")\n\n # Summary\n report.append(\"## Summary\")\n report.append(f\"- **Rows:** {profile.row_count:,}\")\n report.append(f\"- **Columns:** {profile.column_count}\")\n report.append(f\"- **Duplicate Rows:** {profile.duplicate_rows:,}\")\n report.append(f\"- **Memory Usage:** {profile.memory_usage}\")\n report.append(\"\")\n\n # Recommendations\n if profile.recommendations:\n report.append(\"## Recommendations\")\n for rec in profile.recommendations:\n report.append(f\"- {rec}\")\n report.append(\"\")\n\n # Column Details\n report.append(\"## Column Details\")\n report.append(\"\")\n report.append(\"| Column | Type | Inferred | Nulls | Unique | Issues |\")\n report.append(\"|--------|------|----------|-------|--------|--------|\")\n\n for col in profile.columns:\n issues = len(col.quality_issues)\n report.append(\n f\"| {col.name} | {col.data_type} | {col.inferred_type} | \"\n f\"{col.null_percentage:.1f}% | {col.unique_count:,} | {issues} |\"\n )\n\n # Detailed column profiles\n report.append(\"\")\n report.append(\"## Detailed Column Profiles\")\n\n for col in profile.columns:\n report.append(f\"\\n### {col.name}\")\n report.append(f\"- **Type:** {col.data_type} (inferred: {col.inferred_type})\")\n report.append(f\"- **Nulls:** {col.null_count:,} ({col.null_percentage:.1f}%)\")\n report.append(f\"- **Unique Values:** {col.unique_count:,} ({col.uniqueness_ratio:.1%})\")\n\n if col.min_value is not None:\n report.append(f\"- **Range:** {col.min_value:,.2f} to {col.max_value:,.2f}\")\n report.append(f\"- **Mean:** {col.mean_value:,.2f}, Median: {col.median_value:,.2f}\")\n\n if col.min_length is not None:\n report.append(f\"- **Length:** {col.min_length} to {col.max_length} (avg: {col.avg_length:.1f})\")\n\n if col.top_values:\n report.append(f\"- **Top Values:** {col.top_values[:3]}\")\n\n if col.common_patterns:\n report.append(f\"- **Patterns:** {', '.join(col.common_patterns)}\")\n\n if col.quality_issues:\n report.append(f\"- **Issues:** {', '.join(col.quality_issues)}\")\n\n return \"\\n\".join(report)\n\n def compare_profiles(self, profile1: DataProfile, profile2: DataProfile) -> Dict:\n \"\"\"Compare two profiles to detect schema changes or data drift.\"\"\"\n comparison = {\n 'profiles': [profile1.source_name, profile2.source_name],\n 'row_count_change': profile2.row_count - profile1.row_count,\n 'quality_change': profile2.quality_score - profile1.quality_score,\n 'new_columns': [],\n 'removed_columns': [],\n 'type_changes': [],\n 'null_rate_changes': []\n }\n\n cols1 = {c.name: c for c in profile1.columns}\n cols2 = {c.name: c for c in profile2.columns}\n\n # Find new/removed columns\n comparison['new_columns'] = [n for n in cols2 if n not in cols1]\n comparison['removed_columns'] = [n for n in cols1 if n not in cols2]\n\n # Compare common columns\n for name in cols1:\n if name in cols2:\n c1, c2 = cols1[name], cols2[name]\n\n if c1.data_type != c2.data_type:\n comparison['type_changes'].append({\n 'column': name,\n 'from': c1.data_type,\n 'to': c2.data_type\n })\n\n null_change = c2.null_percentage - c1.null_percentage\n if abs(null_change) > 10:\n comparison['null_rate_changes'].append({\n 'column': name,\n 'change': null_change\n })\n\n return comparison\n```\n\n## Quick Start\n\n```python\nimport pandas as pd\n\n# Load construction data\ndf = pd.read_excel(\"project_costs.xlsx\")\n\n# Profile the data\nprofiler = ConstructionDataProfiler()\nprofile = profiler.profile_dataframe(df, \"Project Costs 2025\")\n\n# Generate report\nreport = profiler.generate_profile_report(profile)\nprint(report)\n\n# Export to JSON\nprofile_dict = profiler.profile_to_dict(profile)\nwith open(\"profile.json\", \"w\") as f:\n json.dump(profile_dict, f, indent=2)\n\n# Compare with previous profile\nold_profile = profiler.profile_dataframe(old_df, \"Project Costs 2024\")\ncomparison = profiler.compare_profiles(old_profile, profile)\nprint(f\"Quality changed by: {comparison['quality_change']}\")\n```\n\n## Common Use Cases\n\n1. **Pre-ETL Analysis**: Profile source data before building pipelines\n2. **Quality Monitoring**: Track data quality over time\n3. **Schema Validation**: Detect unexpected changes in data structure\n4. **Anomaly Detection**: Find outliers and data quality issues\n\n## Dependencies\n\n```bash\npip install pandas numpy\n```\n\n## Resources\n\n- **Data Profiling Best Practices**: DAMA DMBOK\n- **Construction Data Standards**: CSI MasterFormat, UniFormat\n---","attachment_filenames":["claw.json","instructions.md"],"attachments":[{"filename":"claw.json","content":"{\n \"name\": \"data-profiler\",\n \"version\": \"2.0.0\",\n \"description\": \"Profile construction data to understand characteristics, distributions, quality metrics, and patterns. Essential for data quality assessment and ETL planning.\",\n \"author\": \"datadrivenconstruction\",\n \"license\": \"MIT\",\n \"permissions\": [\n \"filesystem\"\n ],\n \"entry\": \"instructions.md\",\n \"tags\": [\n \"construction\",\n \"data-processing\",\n \"quality\",\n \"automation\"\n ],\n \"models\": [\n \"claude-*\",\n \"gpt-*\"\n ],\n \"minOpenClawVersion\": \"0.8.0\"\n}","content_type":"application/json; charset=utf-8","language":"json","size":531,"content_sha256":"028d0fcf434bba062d0c9f731f43022e5fa52b5e5696d2ea5a126364b395f658"},{"filename":"instructions.md","content":"You are a construction industry assistant specializing in construction data processing and analytics.\n\nProfile construction data to understand characteristics, distributions, quality metrics, and patterns. Essential for data quality assessment and ETL planning.\n\nWhen the user asks to assist with construction project tasks:\n1. Gather the required input data from the user\n2. Process the data using the methods described in SKILL.md\n3. Present results in a clear, structured format\n4. Offer follow-up analysis or export options\n\n## Input Format\n- The user provides project data, file paths, or parameters as described in SKILL.md\n- Accept data in common formats: CSV, Excel, JSON, or direct input\n\n## Output Format\n- Present results in structured tables when applicable\n- Include summary statistics and key findings\n- Offer export to Excel/CSV/JSON when relevant\n\n## Key Reference\n- See SKILL.md for detailed implementation code, classes, and methods\n- Follow the patterns and APIs defined in the skill documentation\n\n## Constraints\n- Only use data provided by the user or referenced in the skill\n- Validate inputs before processing\n- Report errors clearly with suggested fixes\n- Follow construction industry standards and best practices\n","content_type":"text/markdown; charset=utf-8","language":"markdown","size":1238,"content_sha256":"016f1451c2e0b1df290285d9fdcefbdfa94fd849cc8f64956465a6dab75e264b"}],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Data Profiler for Construction","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Overview","type":"text"}]},{"type":"paragraph","content":[{"text":"Analyze construction data to understand its characteristics, distributions, quality, and patterns. Essential for data quality assessment, ETL planning, and identifying data issues before they impact projects.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Business Case","type":"text"}]},{"type":"paragraph","content":[{"text":"Before using any construction data, you need to understand:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"What data types are present","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Distribution of values","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Missing data patterns","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Anomalies and outliers","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Referential integrity issues","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"This skill profiles data to answer these questions and provides actionable insights.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Technical Implementation","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from dataclasses import dataclass, field\nfrom typing import List, Dict, Any, Optional, Tuple\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport json\n\n@dataclass\nclass ColumnProfile:\n name: str\n data_type: str\n inferred_type: str # More specific: project_id, cost, date, csi_code, etc.\n total_count: int\n null_count: int\n null_percentage: float\n unique_count: int\n uniqueness_ratio: float\n # For numeric columns\n min_value: Optional[float] = None\n max_value: Optional[float] = None\n mean_value: Optional[float] = None\n median_value: Optional[float] = None\n std_dev: Optional[float] = None\n # For string columns\n min_length: Optional[int] = None\n max_length: Optional[int] = None\n avg_length: Optional[float] = None\n # Top values\n top_values: List[Tuple[Any, int]] = field(default_factory=list)\n # Patterns\n common_patterns: List[str] = field(default_factory=list)\n # Quality flags\n quality_issues: List[str] = field(default_factory=list)\n\n@dataclass\nclass DataProfile:\n source_name: str\n row_count: int\n column_count: int\n columns: List[ColumnProfile]\n duplicate_rows: int\n memory_usage: str\n profiled_at: datetime\n quality_score: float\n recommendations: List[str]\n\nclass ConstructionDataProfiler:\n \"\"\"Profile construction data for quality and characteristics.\"\"\"\n\n # Known construction data patterns\n CONSTRUCTION_PATTERNS = {\n 'csi_code': r'^\\d{2}\\s?\\d{2}\\s?\\d{2}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'project_id': r'^[A-Z]{2,4}[-_]?\\d{3,6}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'cost_code': r'^\\d{2}[-.]?\\d{2,4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'wbs': r'^[\\d.]+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'phone': r'^\$?\\d{3}\$?[-.\\s]?\\d{3}[-.\\s]?\\d{4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'email': r'^[\\w.-]+@[\\w.-]+\\.\\w+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'date_iso': r'^\\d{4}-\\d{2}-\\d{2}',\n 'date_us': r'^\\d{1,2}/\\d{1,2}/\\d{2,4}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'currency': r'^\\$?[\\d,]+\\.?\\d{0,2}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n 'percentage': r'^\\d+\\.?\\d*%?

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.

,\n }\n\n # Construction-specific column name patterns\n COLUMN_TYPE_HINTS = {\n 'project': ['project_id', 'project_name', 'proj', 'job'],\n 'cost': ['cost', 'amount', 'price', 'total', 'budget', 'actual'],\n 'date': ['date', 'start', 'finish', 'end', 'created', 'modified'],\n 'quantity': ['qty', 'quantity', 'count', 'units'],\n 'csi': ['csi', 'division', 'masterformat', 'spec'],\n 'location': ['location', 'area', 'zone', 'floor', 'level'],\n 'person': ['owner', 'manager', 'superintendent', 'foreman', 'contact'],\n }\n\n def __init__(self):\n self.profiles: Dict[str, DataProfile] = {}\n\n def profile_dataframe(self, df: pd.DataFrame, source_name: str) -> DataProfile:\n \"\"\"Profile a pandas DataFrame.\"\"\"\n columns = []\n\n for col in df.columns:\n col_profile = self._profile_column(df[col], col)\n columns.append(col_profile)\n\n # Calculate duplicates\n duplicate_rows = len(df) - len(df.drop_duplicates())\n\n # Calculate memory usage\n memory_bytes = df.memory_usage(deep=True).sum()\n if memory_bytes \u003c 1024:\n memory_usage = f\"{memory_bytes} B\"\n elif memory_bytes \u003c 1024**2:\n memory_usage = f\"{memory_bytes/1024:.1f} KB\"\n else:\n memory_usage = f\"{memory_bytes/1024**2:.1f} MB\"\n\n # Calculate overall quality score\n quality_score = self._calculate_quality_score(columns)\n\n # Generate recommendations\n recommendations = self._generate_recommendations(columns, df)\n\n profile = DataProfile(\n source_name=source_name,\n row_count=len(df),\n column_count=len(df.columns),\n columns=columns,\n duplicate_rows=duplicate_rows,\n memory_usage=memory_usage,\n profiled_at=datetime.now(),\n quality_score=quality_score,\n recommendations=recommendations\n )\n\n self.profiles[source_name] = profile\n return profile\n\n def _profile_column(self, series: pd.Series, name: str) -> ColumnProfile:\n \"\"\"Profile a single column.\"\"\"\n total_count = len(series)\n null_count = series.isnull().sum()\n null_percentage = (null_count / total_count * 100) if total_count > 0 else 0\n\n # Get non-null values for analysis\n non_null = series.dropna()\n unique_count = non_null.nunique()\n uniqueness_ratio = unique_count / len(non_null) if len(non_null) > 0 else 0\n\n profile = ColumnProfile(\n name=name,\n data_type=str(series.dtype),\n inferred_type=self._infer_construction_type(series, name),\n total_count=total_count,\n null_count=null_count,\n null_percentage=round(null_percentage, 2),\n unique_count=unique_count,\n uniqueness_ratio=round(uniqueness_ratio, 4)\n )\n\n # Numeric analysis\n if pd.api.types.is_numeric_dtype(series):\n profile.min_value = float(non_null.min()) if len(non_null) > 0 else None\n profile.max_value = float(non_null.max()) if len(non_null) > 0 else None\n profile.mean_value = float(non_null.mean()) if len(non_null) > 0 else None\n profile.median_value = float(non_null.median()) if len(non_null) > 0 else None\n profile.std_dev = float(non_null.std()) if len(non_null) > 1 else None\n\n # Check for outliers\n if len(non_null) > 10 and profile.std_dev:\n outliers = non_null[abs(non_null - profile.mean_value) > 3 * profile.std_dev]\n if len(outliers) > 0:\n profile.quality_issues.append(f\"{len(outliers)} potential outliers detected\")\n\n # Check for negative costs\n if any(hint in name.lower() for hint in ['cost', 'amount', 'price', 'total']):\n negatives = (non_null \u003c 0).sum()\n if negatives > 0:\n profile.quality_issues.append(f\"{negatives} negative values in cost column\")\n\n # String analysis\n elif pd.api.types.is_object_dtype(series) or pd.api.types.is_string_dtype(series):\n str_series = non_null.astype(str)\n lengths = str_series.str.len()\n profile.min_length = int(lengths.min()) if len(lengths) > 0 else None\n profile.max_length = int(lengths.max()) if len(lengths) > 0 else None\n profile.avg_length = float(lengths.mean()) if len(lengths) > 0 else None\n\n # Detect patterns\n profile.common_patterns = self._detect_patterns(str_series)\n\n # Top values\n if len(non_null) > 0:\n value_counts = non_null.value_counts().head(5)\n profile.top_values = list(zip(value_counts.index.tolist(), value_counts.values.tolist()))\n\n # Quality checks\n if null_percentage > 50:\n profile.quality_issues.append(\"High null rate (>50%)\")\n if uniqueness_ratio == 1.0 and total_count > 100:\n profile.quality_issues.append(\"All unique values - possible ID column\")\n if uniqueness_ratio \u003c 0.01 and unique_count > 1:\n profile.quality_issues.append(\"Low cardinality - possible category\")\n\n return profile\n\n def _infer_construction_type(self, series: pd.Series, name: str) -> str:\n \"\"\"Infer construction-specific data type.\"\"\"\n name_lower = name.lower()\n\n # Check column name hints\n for type_name, hints in self.COLUMN_TYPE_HINTS.items():\n if any(hint in name_lower for hint in hints):\n return type_name\n\n # Check data patterns\n non_null = series.dropna().astype(str)\n if len(non_null) == 0:\n return \"unknown\"\n\n sample = non_null.head(100)\n\n for pattern_name, pattern in self.CONSTRUCTION_PATTERNS.items():\n matches = sample.str.match(pattern, na=False).sum()\n if matches / len(sample) > 0.8:\n return pattern_name\n\n # Default to pandas dtype\n if pd.api.types.is_numeric_dtype(series):\n return \"numeric\"\n elif pd.api.types.is_datetime64_any_dtype(series):\n return \"datetime\"\n else:\n return \"text\"\n\n def _detect_patterns(self, str_series: pd.Series) -> List[str]:\n \"\"\"Detect common patterns in string data.\"\"\"\n patterns_found = []\n\n sample = str_series.head(1000)\n\n for pattern_name, pattern in self.CONSTRUCTION_PATTERNS.items():\n matches = sample.str.match(pattern, na=False).sum()\n if matches / len(sample) > 0.1:\n patterns_found.append(f\"{pattern_name} ({matches/len(sample):.0%})\")\n\n return patterns_found[:3]\n\n def _calculate_quality_score(self, columns: List[ColumnProfile]) -> float:\n \"\"\"Calculate overall data quality score (0-100).\"\"\"\n if not columns:\n return 0.0\n\n scores = []\n\n for col in columns:\n col_score = 100\n\n # Penalize for nulls\n col_score -= min(col.null_percentage, 50)\n\n # Penalize for quality issues\n col_score -= len(col.quality_issues) * 10\n\n scores.append(max(col_score, 0))\n\n return round(sum(scores) / len(scores), 1)\n\n def _generate_recommendations(self, columns: List[ColumnProfile], df: pd.DataFrame) -> List[str]:\n \"\"\"Generate recommendations based on profile.\"\"\"\n recommendations = []\n\n # High null columns\n high_null = [c for c in columns if c.null_percentage > 30]\n if high_null:\n recommendations.append(\n f\"Review {len(high_null)} columns with >30% null values: \"\n f\"{', '.join(c.name for c in high_null[:3])}\"\n )\n\n # Potential ID columns without uniqueness\n for col in columns:\n if 'id' in col.name.lower() and col.uniqueness_ratio \u003c 1.0:\n recommendations.append(\n f\"Column '{col.name}' appears to be an ID but has duplicate values\"\n )\n\n # Date columns that should be datetime\n for col in columns:\n if col.inferred_type in ['date_iso', 'date_us'] and col.data_type == 'object':\n recommendations.append(\n f\"Convert '{col.name}' to datetime type for better analysis\"\n )\n\n # Cost columns that are strings\n for col in columns:\n if col.inferred_type == 'currency' and col.data_type == 'object':\n recommendations.append(\n f\"Convert '{col.name}' to numeric type (remove $ and commas)\"\n )\n\n return recommendations\n\n def profile_to_dict(self, profile: DataProfile) -> Dict:\n \"\"\"Convert profile to dictionary for JSON export.\"\"\"\n return {\n 'source_name': profile.source_name,\n 'row_count': profile.row_count,\n 'column_count': profile.column_count,\n 'duplicate_rows': profile.duplicate_rows,\n 'memory_usage': profile.memory_usage,\n 'profiled_at': profile.profiled_at.isoformat(),\n 'quality_score': profile.quality_score,\n 'recommendations': profile.recommendations,\n 'columns': [\n {\n 'name': c.name,\n 'data_type': c.data_type,\n 'inferred_type': c.inferred_type,\n 'null_percentage': c.null_percentage,\n 'unique_count': c.unique_count,\n 'quality_issues': c.quality_issues,\n 'top_values': c.top_values[:3]\n }\n for c in profile.columns\n ]\n }\n\n def generate_profile_report(self, profile: DataProfile) -> str:\n \"\"\"Generate markdown profile report.\"\"\"\n report = [f\"# Data Profile: {profile.source_name}\", \"\"]\n report.append(f\"**Profiled At:** {profile.profiled_at.strftime('%Y-%m-%d %H:%M')}\")\n report.append(f\"**Quality Score:** {profile.quality_score}/100\")\n report.append(\"\")\n\n # Summary\n report.append(\"## Summary\")\n report.append(f\"- **Rows:** {profile.row_count:,}\")\n report.append(f\"- **Columns:** {profile.column_count}\")\n report.append(f\"- **Duplicate Rows:** {profile.duplicate_rows:,}\")\n report.append(f\"- **Memory Usage:** {profile.memory_usage}\")\n report.append(\"\")\n\n # Recommendations\n if profile.recommendations:\n report.append(\"## Recommendations\")\n for rec in profile.recommendations:\n report.append(f\"- {rec}\")\n report.append(\"\")\n\n # Column Details\n report.append(\"## Column Details\")\n report.append(\"\")\n report.append(\"| Column | Type | Inferred | Nulls | Unique | Issues |\")\n report.append(\"|--------|------|----------|-------|--------|--------|\")\n\n for col in profile.columns:\n issues = len(col.quality_issues)\n report.append(\n f\"| {col.name} | {col.data_type} | {col.inferred_type} | \"\n f\"{col.null_percentage:.1f}% | {col.unique_count:,} | {issues} |\"\n )\n\n # Detailed column profiles\n report.append(\"\")\n report.append(\"## Detailed Column Profiles\")\n\n for col in profile.columns:\n report.append(f\"\\n### {col.name}\")\n report.append(f\"- **Type:** {col.data_type} (inferred: {col.inferred_type})\")\n report.append(f\"- **Nulls:** {col.null_count:,} ({col.null_percentage:.1f}%)\")\n report.append(f\"- **Unique Values:** {col.unique_count:,} ({col.uniqueness_ratio:.1%})\")\n\n if col.min_value is not None:\n report.append(f\"- **Range:** {col.min_value:,.2f} to {col.max_value:,.2f}\")\n report.append(f\"- **Mean:** {col.mean_value:,.2f}, Median: {col.median_value:,.2f}\")\n\n if col.min_length is not None:\n report.append(f\"- **Length:** {col.min_length} to {col.max_length} (avg: {col.avg_length:.1f})\")\n\n if col.top_values:\n report.append(f\"- **Top Values:** {col.top_values[:3]}\")\n\n if col.common_patterns:\n report.append(f\"- **Patterns:** {', '.join(col.common_patterns)}\")\n\n if col.quality_issues:\n report.append(f\"- **Issues:** {', '.join(col.quality_issues)}\")\n\n return \"\\n\".join(report)\n\n def compare_profiles(self, profile1: DataProfile, profile2: DataProfile) -> Dict:\n \"\"\"Compare two profiles to detect schema changes or data drift.\"\"\"\n comparison = {\n 'profiles': [profile1.source_name, profile2.source_name],\n 'row_count_change': profile2.row_count - profile1.row_count,\n 'quality_change': profile2.quality_score - profile1.quality_score,\n 'new_columns': [],\n 'removed_columns': [],\n 'type_changes': [],\n 'null_rate_changes': []\n }\n\n cols1 = {c.name: c for c in profile1.columns}\n cols2 = {c.name: c for c in profile2.columns}\n\n # Find new/removed columns\n comparison['new_columns'] = [n for n in cols2 if n not in cols1]\n comparison['removed_columns'] = [n for n in cols1 if n not in cols2]\n\n # Compare common columns\n for name in cols1:\n if name in cols2:\n c1, c2 = cols1[name], cols2[name]\n\n if c1.data_type != c2.data_type:\n comparison['type_changes'].append({\n 'column': name,\n 'from': c1.data_type,\n 'to': c2.data_type\n })\n\n null_change = c2.null_percentage - c1.null_percentage\n if abs(null_change) > 10:\n comparison['null_rate_changes'].append({\n 'column': name,\n 'change': null_change\n })\n\n return comparison","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Quick Start","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"import pandas as pd\n\n# Load construction data\ndf = pd.read_excel(\"project_costs.xlsx\")\n\n# Profile the data\nprofiler = ConstructionDataProfiler()\nprofile = profiler.profile_dataframe(df, \"Project Costs 2025\")\n\n# Generate report\nreport = profiler.generate_profile_report(profile)\nprint(report)\n\n# Export to JSON\nprofile_dict = profiler.profile_to_dict(profile)\nwith open(\"profile.json\", \"w\") as f:\n json.dump(profile_dict, f, indent=2)\n\n# Compare with previous profile\nold_profile = profiler.profile_dataframe(old_df, \"Project Costs 2024\")\ncomparison = profiler.compare_profiles(old_profile, profile)\nprint(f\"Quality changed by: {comparison['quality_change']}\")","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Common Use Cases","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Pre-ETL Analysis","type":"text","marks":[{"type":"strong"}]},{"text":": Profile source data before building pipelines","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Quality Monitoring","type":"text","marks":[{"type":"strong"}]},{"text":": Track data quality over time","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Schema Validation","type":"text","marks":[{"type":"strong"}]},{"text":": Detect unexpected changes in data structure","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Anomaly Detection","type":"text","marks":[{"type":"strong"}]},{"text":": Find outliers and data quality issues","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Dependencies","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"pip install pandas numpy","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Resources","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Data Profiling Best Practices","type":"text","marks":[{"type":"strong"}]},{"text":": DAMA DMBOK","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Construction Data Standards","type":"text","marks":[{"type":"strong"}]},{"text":": CSI MasterFormat, UniFormat","type":"text"}]}]}]},{"type":"hr","attrs":{"markup":"---"}}]},"metadata":{"date":"2026-06-05","name":"data-profiler","author":"@skillopedia","source":{"stars":155,"repo_name":"ddc_skills_for_ai_agents_in_construction","origin_url":"https://github.com/datadrivenconstruction/ddc_skills_for_ai_agents_in_construction/blob/HEAD/2_DDC_Book/2.1-Data-Types-Classification/data-profiler/SKILL.md","repo_owner":"datadrivenconstruction","body_sha256":"4c90113dbcd3de3f8340a112e79d85a18b82f51bd7ae567d740a71646bea5f42","cluster_key":"f19443e988bcc8f3a0f74069b4c7caf5a4811742372f89faf990374a4b354a45","clean_bundle":{"format":"clean-skill-bundle-v1","source":"datadrivenconstruction/ddc_skills_for_ai_agents_in_construction/2_DDC_Book/2.1-Data-Types-Classification/data-profiler/SKILL.md","attachments":[{"id":"66662df4-e7b4-57b1-a25a-984469d2722d","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/66662df4-e7b4-57b1-a25a-984469d2722d/attachment.json","path":"claw.json","size":531,"sha256":"028d0fcf434bba062d0c9f731f43022e5fa52b5e5696d2ea5a126364b395f658","contentType":"application/json; charset=utf-8"},{"id":"de51d3d7-98c8-5468-b378-1163505c3030","key":"uploads/10433ee7-ad12-4ae0-b34e-97553e46c6c8/de51d3d7-98c8-5468-b378-1163505c3030/attachment.md","path":"instructions.md","size":1238,"sha256":"016f1451c2e0b1df290285d9fdcefbdfa94fd849cc8f64956465a6dab75e264b","contentType":"text/markdown; charset=utf-8"}],"bundle_sha256":"60329c10290f955a64f4792ca19f690f832b7dcffbd2bb28ad143c9d9d02a1de","attachment_count":2,"text_attachments":2,"attachment_storage":"skillopedia-attachments-v1","binary_attachments":0,"excluded_attachments":[]},"cluster_size":1,"skill_md_path":"2_DDC_Book/2.1-Data-Types-Classification/data-profiler/SKILL.md","import_metadata":{"date":"2026-06-05","author":"@skillopedia","version":"v1","category":"data-analytics","category_label":"Data"},"exact_dupes_collapsed_into_this":0},"version":"v1","category":"data-analytics","homepage":"https://datadrivenconstruction.io","metadata":{"openclaw":{"os":["darwin","linux","win32"],"emoji":"🏷️","homepage":"https://datadrivenconstruction.io","requires":{"bins":["python3"]}}},"import_tag":"clean-skills-v1","description":"Profile construction data to understand characteristics, distributions, quality metrics, and patterns. Essential for data quality assessment and ETL planning."}},"renderedAt":1782979501147}

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.