data-validation-pattern

Data Validation Security Pattern Ensures all incoming data is validated against specifications before processing, preventing injection attacks, data corruption, and unexpected behavior. When to Use Use this pattern when: - Processing ANY input from external sources (users, APIs, databases) - Preventing injection attacks (SQLi, XSS, Command Injection) - Implementing API request validation checklists - Ensuring data integrity for business logic - Handling file uploads or complex data structures Problem Addressed Entity provides unexpected data : Malicious or malformed input causes: - Injection…

)\n email: EmailStr\n age: conint(ge=18, le=120)\n\[email protected](\"/user\", methods=[\"POST\"])\ndef create_user():\n try:\n # ✅ Validate payload against schema\n user = UserSchema(**request.get_json())\n save_to_db(user.model_dump())\n except ValueError as e:\n return jsonify({\"error\": str(e)}), 400\n```\n\n### JavaScript (Zod / Express)\n\n**BAD (Vulnerable):**\n\n```javascript\n// ❌ VULNERABILITY: Implicit trust\napp.post('/api/profile', (req, res) => {\n // trusting req.body.website is a valid URL\n // trusting req.body.role is not \"admin\"\n updateProfile(req.user.id, req.body);\n});\n```\n\n**GOOD (Secure):**\n\n```javascript\nconst { z } = require('zod');\n\n// ✅ Define strict schema\nconst ProfileSchema = z.object({\n website: z.string().url().max(100),\n bio: z.string().max(500).optional(),\n role: z.enum(['user', 'editor']), // Block 'admin'\n});\n\napp.post('/api/profile', (req, res) => {\n const result = ProfileSchema.safeParse(req.body);\n\n if (!result.success) {\n return res.status(400).json(result.error);\n }\n\n // ✅ Apply canonical/validated data\n updateProfile(req.user.id, result.data);\n});\n```\n\n## Implementation Checklist\n\n- [ ] All entry points have validation\n- [ ] Canonical form transformation\n- [ ] Allowlist-based rules\n- [ ] Type checking\n- [ ] Length/range limits\n- [ ] Business rule validation\n- [ ] Secure error handling\n- [ ] Output encoding (separate from validation)\n- [ ] File upload validation\n- [ ] Structured data parsing safely\n- [ ] Re-validation near sensitive operations\n\n## Related Patterns\n\n- Authorisation (validation doesn't replace authorization)\n- Selective encrypted transmission (protect data in transit)\n- Log entity actions (log validation failures)\n\n## References\n\n- Source: \u003chttps://securitypatterns.distrinet-research.be/patterns/04_01_001__data_validation/>\n- OWASP Input Validation Cheat Sheet\n- OWASP XSS Prevention Cheat Sheet\n---","attachment_filenames":[],"attachments":[],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Data Validation Security Pattern","type":"text"}]},{"type":"paragraph","content":[{"text":"Ensures all incoming data is validated against specifications before processing, preventing injection attacks, data corruption, and unexpected behavior.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"When to Use","type":"text"}]},{"type":"paragraph","content":[{"text":"Use this pattern when:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Processing ANY input from external sources (users, APIs, databases)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Preventing injection attacks (SQLi, XSS, Command Injection)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Implementing API request validation checklists","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Ensuring data integrity for business logic","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Handling file uploads or complex data structures","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Problem Addressed","type":"text"}]},{"type":"paragraph","content":[{"text":"Entity provides unexpected data","type":"text","marks":[{"type":"strong"}]},{"text":": Malicious or malformed input causes:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Injection attacks (SQL, XSS, command injection)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"System crashes or unexpected behavior","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Data corruption","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Security bypasses","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Core Components","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Role","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Type","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Responsibility","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Entity","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Entity","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Sends data to system","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Enforcer","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Enforcement Point","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Intercepts all incoming data","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Validator","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Decision Point","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Validates data against specification","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Specification Provider","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Information Point","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Manages validation rules","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"System","type":"text","marks":[{"type":"strong"}]}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Entity","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Processes validated data","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Data Elements","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"data","type":"text","marks":[{"type":"strong"}]},{"text":": Input from entity (raw)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"canonical_data","type":"text","marks":[{"type":"strong"}]},{"text":": Normalized, validated form","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"specification","type":"text","marks":[{"type":"strong"}]},{"text":": Rules defining valid data","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"type","type":"text","marks":[{"type":"strong"}]},{"text":": Identifier for applicable specification","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"error","type":"text","marks":[{"type":"strong"}]},{"text":": Validation failure message","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Validation Flow","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":""},"content":[{"text":"Entity → [data] → Enforcer\nEnforcer → [data] → Validator\nValidator → [get_specification(type)] → Specification Provider\nSpecification Provider → [specification] → Validator\nValidator → [validate, transform to canonical] → Validator\nValidator → [canonical_data or error] → Enforcer\nEnforcer → [canonical_data] → System (if valid)\n → [error] → Entity (if invalid)","type":"text"}]},{"type":"ordered_list","attrs":{"order":1,"listStyle":"number"},"content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Enforcer intercepts ALL incoming data","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validator retrieves appropriate specification","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validator transforms to canonical form","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validator checks against specification","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Valid: canonical data forwarded to System","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Invalid: error returned to Entity","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Validation Principles","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Validate Everything","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"All data from uncontrolled sources","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Parameters, headers, cookies, files","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Data from APIs, databases (defense in depth)","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Canonical Form","type":"text"}]},{"type":"paragraph","content":[{"text":"Transform data to standardized form:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Remove/escape special characters","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Decode encoded values","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Normalize Unicode","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Parse structured data to typed objects","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Benefit","type":"text","marks":[{"type":"strong"}]},{"text":": System only processes data in known format.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Allowlist vs. Blocklist","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Allowlist (preferred)","type":"text","marks":[{"type":"strong"}]},{"text":": Define what IS allowed","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Blocklist (risky)","type":"text","marks":[{"type":"strong"}]},{"text":": Define what is NOT allowed","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Blocklists fail against unknown attack patterns. Use allowlists.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Validate Early, Validate Often","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validate at system boundary (earliest point)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Re-validate near code that relies on data","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Defense in depth","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Validation Types","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Type Validation","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Ensure data matches expected type","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Integer, string, boolean, date, email, URL","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Range/Length Validation","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Numeric bounds","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"String length limits","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Array size limits","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Format Validation","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Regular expressions (carefully!)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Structural patterns","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Protocol conformance","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Business Logic Validation","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Application-specific rules","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Cross-field validation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"State-dependent validation","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Security Considerations","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Validation ≠ Authorization","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validation: Is this data well-formed?","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Authorization: Is entity allowed to use this data?","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Both are required. Valid data doesn't mean authorized access.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Error Messages","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Don't reveal validation internals to attackers","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Log detailed errors server-side","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Return generic errors to clients","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Encoding Output","type":"text"}]},{"type":"paragraph","content":[{"text":"Validation alone doesn't prevent all injection:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Still encode output for context (HTML, SQL, etc.)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Use parameterized queries","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Use context-appropriate escaping","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"File Uploads","type":"text"}]},{"type":"paragraph","content":[{"text":"Special validation needed:","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Verify content type (not just extension)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Scan for malware","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Restrict file sizes","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Store outside web root","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Structured Data (JSON, XML)","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Parse with secure parser","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Disable external entity processing (XXE)","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Validate against schema","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Limit nesting depth","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Regular Expression Safety","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Avoid ReDoS-vulnerable patterns","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Limit input length before regex","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Test regex performance with malicious input","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Common Validation Scenarios","type":"text"}]},{"type":"table","attrs":{"layout":null},"content":[{"type":"tr","content":[{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Input Type","type":"text"}]}]},{"type":"th","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Validations","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Username","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Length, allowed characters, no control chars","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Email","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Format, length, allowlist domains (if applicable)","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Integer","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Type, range, positive/negative","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"URL","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Protocol allowlist, format, no javascript:","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"File","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Extension, content-type, size, malware scan","type":"text"}]}]}]},{"type":"tr","content":[{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"JSON","type":"text"}]}]},{"type":"td","attrs":{"colspan":1,"rowspan":1,"colwidth":null,"alignment":""},"content":[{"type":"paragraph","content":[{"text":"Schema validation, depth limits, size limits","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Implementation Examples","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Python (Pydantic / Flask)","type":"text"}]},{"type":"paragraph","content":[{"text":"BAD (Vulnerable):","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# ❌ VULNERABILITY: Manual, incomplete validation\[email protected](\"/user\", methods=[\"POST\"])\ndef create_user():\n data = request.get_json()\n if 'email' not in data: # What about type? Length? format?\n return \"Missing email\", 400\n # ... proceeding to use data['age'] which might be a string or negative","type":"text"}]},{"type":"paragraph","content":[{"text":"GOOD (Secure):","type":"text","marks":[{"type":"strong"}]}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from pydantic import BaseModel, EmailStr, conint, constr\n\n# ✅ Define strict schema\nclass UserSchema(BaseModel):\n username: constr(min_length=3, max_length=50, pattern=r'^[a-zA-Z0-9_]+

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.