beautifulsoup-parsing — Skillopedia

BeautifulSoup HTML Parsing You are an expert in BeautifulSoup, Python HTML/XML parsing, DOM navigation, and building efficient data extraction pipelines for web scraping. Core Expertise - BeautifulSoup API and parsing methods - CSS selectors and find methods - DOM traversal and navigation - HTML/XML parsing with different parsers - Integration with requests library - Handling malformed HTML gracefully - Data extraction patterns and best practices - Memory-efficient processing Key Principles - Write concise, technical code with accurate Python examples - Prioritize readability, efficiency, and…

, '').replace(',', ''))\n except ValueError:\n return None\n\n def _resolve(self, url):\n return urljoin(self.base_url, url) if url else None\n\n\n# Usage\nscraper = ProductScraper('https://example.com')\nproducts = scraper.scrape_products('https://example.com/products')\nfor product in products:\n print(product)\n```\n\n## Performance Optimization\n\n```python\n# Use SoupStrainer to parse only needed elements\nfrom bs4 import SoupStrainer\n\nonly_articles = SoupStrainer('article')\nsoup = BeautifulSoup(html, 'lxml', parse_only=only_articles)\n\n# Use lxml parser for speed\nsoup = BeautifulSoup(html, 'lxml') # Fastest\n\n# Decompose unneeded elements\nfor script in soup.find_all('script'):\n script.decompose()\n\n# Use generators for memory efficiency\nfor item in soup.select('.item'):\n yield extract_data(item)\n```\n\n## Key Dependencies\n\n- beautifulsoup4\n- lxml (fast parser)\n- html5lib (lenient parser)\n- requests\n- pandas (for data output)\n\n## Best Practices\n\n1. Always use lxml parser for best performance\n2. Handle missing elements with default values\n3. Use `select()` and `select_one()` for CSS selectors\n4. Use `get_text(strip=True)` for clean text extraction\n5. Resolve relative URLs to absolute\n6. Validate extracted data types\n7. Implement rate limiting between requests\n8. Use proper User-Agent headers\n9. Handle character encoding properly\n10. Use SoupStrainer for large documents\n11. Follow robots.txt and website terms of service\n12. Implement retry logic for failed requests","attachment_filenames":[],"attachments":[],"content_json":{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"BeautifulSoup HTML Parsing","type":"text"}]},{"type":"paragraph","content":[{"text":"You are an expert in BeautifulSoup, Python HTML/XML parsing, DOM navigation, and building efficient data extraction pipelines for web scraping.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Core Expertise","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"BeautifulSoup API and parsing methods","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"CSS selectors and find methods","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"DOM traversal and navigation","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"HTML/XML parsing with different parsers","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Integration with requests library","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Handling malformed HTML gracefully","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Data extraction patterns and best practices","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Memory-efficient processing","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Key Principles","type":"text"}]},{"type":"bullet_list","content":[{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Write concise, technical code with accurate Python examples","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Prioritize readability, efficiency, and maintainability","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Use modular, reusable functions for common extraction tasks","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Handle missing data gracefully with proper defaults","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Follow PEP 8 style guidelines","type":"text"}]}]},{"type":"list_item","content":[{"type":"paragraph","content":[{"text":"Implement proper error handling for robust scraping","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Basic Setup","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"bash"},"content":[{"text":"pip install beautifulsoup4 requests lxml","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Loading HTML","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from bs4 import BeautifulSoup\nimport requests\n\n# From string\nhtml = '\u003chtml>\u003cbody>\u003ch1>Hello\u003c/h1>\u003c/body>\u003c/html>'\nsoup = BeautifulSoup(html, 'lxml')\n\n# From file\nwith open('page.html', 'r', encoding='utf-8') as f:\n soup = BeautifulSoup(f, 'lxml')\n\n# From URL\nresponse = requests.get('https://example.com')\nsoup = BeautifulSoup(response.content, 'lxml')","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Parser Options","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# lxml - Fast, lenient (recommended)\nsoup = BeautifulSoup(html, 'lxml')\n\n# html.parser - Built-in, no dependencies\nsoup = BeautifulSoup(html, 'html.parser')\n\n# html5lib - Most lenient, slowest\nsoup = BeautifulSoup(html, 'html5lib')\n\n# lxml-xml - For XML documents\nsoup = BeautifulSoup(xml, 'lxml-xml')","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Finding Elements","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"By Tag","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# First matching element\nsoup.find('h1')\n\n# All matching elements\nsoup.find_all('p')\n\n# Shorthand\nsoup.h1 # Same as soup.find('h1')","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"By Attributes","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# By class\nsoup.find('div', class_='article')\nsoup.find_all('div', class_='article')\n\n# By ID\nsoup.find(id='main-content')\n\n# By any attribute\nsoup.find('a', href='https://example.com')\nsoup.find_all('input', attrs={'type': 'text', 'name': 'email'})\n\n# By data attributes\nsoup.find('div', attrs={'data-id': '123'})","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"CSS Selectors","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# Single element\nsoup.select_one('div.article > h2')\n\n# Multiple elements\nsoup.select('div.article h2')\n\n# Complex selectors\nsoup.select('a[href^=\"https://\"]') # Starts with\nsoup.select('a[href$=\".pdf\"]') # Ends with\nsoup.select('a[href*=\"example\"]') # Contains\nsoup.select('li:nth-child(2)')\nsoup.select('h1, h2, h3') # Multiple","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"With Functions","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"import re\n\n# By regex\nsoup.find_all('a', href=re.compile(r'^https://'))\n\n# By function\ndef has_data_attr(tag):\n return tag.has_attr('data-id')\n\nsoup.find_all(has_data_attr)\n\n# String matching\nsoup.find_all(string='exact text')\nsoup.find_all(string=re.compile('pattern'))","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Extracting Data","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Text Content","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# Get text\nelement.text\nelement.get_text()\n\n# Get text with separator\nelement.get_text(separator=' ')\n\n# Get stripped text\nelement.get_text(strip=True)\n\n# Get strings (generator)\nfor string in element.stripped_strings:\n print(string)","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Attributes","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# Get attribute\nelement['href']\nelement.get('href') # Returns None if missing\nelement.get('href', 'default') # With default\n\n# Get all attributes\nelement.attrs # Returns dict\n\n# Check attribute exists\nelement.has_attr('class')","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"HTML Content","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# Inner HTML\nstr(element)\n\n# Just the tag\nelement.name\n\n# Prettified HTML\nelement.prettify()","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"DOM Navigation","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Parent/Ancestors","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"element.parent\nelement.parents # Generator of all ancestors\n\n# Find specific ancestor\nfor parent in element.parents:\n if parent.name == 'div' and 'article' in parent.get('class', []):\n break","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Children","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"element.children # Direct children (generator)\nlist(element.children)\n\nelement.contents # Direct children (list)\nelement.descendants # All descendants (generator)\n\n# Find in children\nelement.find('span') # Searches descendants","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Siblings","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"element.next_sibling\nelement.previous_sibling\n\nelement.next_siblings # Generator\nelement.previous_siblings # Generator\n\n# Next/previous element (skips whitespace)\nelement.next_element\nelement.previous_element","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Data Extraction Patterns","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Safe Extraction","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"def safe_text(element, selector, default=''):\n \"\"\"Safely extract text from element.\"\"\"\n found = element.select_one(selector)\n return found.get_text(strip=True) if found else default\n\ndef safe_attr(element, selector, attr, default=None):\n \"\"\"Safely extract attribute from element.\"\"\"\n found = element.select_one(selector)\n return found.get(attr, default) if found else default","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Table Extraction","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"def extract_table(table):\n \"\"\"Extract table data as list of dictionaries.\"\"\"\n headers = [th.get_text(strip=True) for th in table.select('th')]\n\n rows = []\n for tr in table.select('tbody tr'):\n cells = [td.get_text(strip=True) for td in tr.select('td')]\n if cells:\n rows.append(dict(zip(headers, cells)))\n\n return rows","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"List Extraction","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"def extract_items(soup, selector, extractor):\n \"\"\"Extract multiple items using a custom extractor function.\"\"\"\n return [extractor(item) for item in soup.select(selector)]\n\n# Usage\ndef extract_product(item):\n return {\n 'name': safe_text(item, '.name'),\n 'price': safe_text(item, '.price'),\n 'url': safe_attr(item, 'a', 'href')\n }\n\nproducts = extract_items(soup, '.product', extract_product)","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"URL Resolution","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"from urllib.parse import urljoin\n\ndef resolve_url(base_url, relative_url):\n \"\"\"Convert relative URL to absolute.\"\"\"\n if not relative_url:\n return None\n return urljoin(base_url, relative_url)\n\n# Usage\nbase_url = 'https://example.com/products/'\nfor link in soup.select('a'):\n href = link.get('href')\n absolute_url = resolve_url(base_url, href)\n print(absolute_url)","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Handling Malformed HTML","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"# lxml parser is lenient with malformed HTML\nsoup = BeautifulSoup(malformed_html, 'lxml')\n\n# For very broken HTML, use html5lib\nsoup = BeautifulSoup(very_broken_html, 'html5lib')\n\n# Handle encoding issues\nresponse = requests.get(url)\nresponse.encoding = response.apparent_encoding\nsoup = BeautifulSoup(response.text, 'lxml')","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Complete Scraping Example","type":"text"}]},{"type":"code_block","attrs":{"wrap":false,"language":"python"},"content":[{"text":"import requests\nfrom bs4 import BeautifulSoup\nfrom urllib.parse import urljoin\nimport time\n\nclass ProductScraper:\n def __init__(self, base_url):\n self.base_url = base_url\n self.session = requests.Session()\n self.session.headers.update({\n 'User-Agent': 'Mozilla/5.0 (compatible; MyScraper/1.0)'\n })\n\n def fetch_page(self, url):\n \"\"\"Fetch and parse a page.\"\"\"\n response = self.session.get(url, timeout=30)\n response.raise_for_status()\n return BeautifulSoup(response.content, 'lxml')\n\n def extract_product(self, item):\n \"\"\"Extract product data from a card element.\"\"\"\n return {\n 'name': self._safe_text(item, '.product-title'),\n 'price': self._parse_price(item.select_one('.price')),\n 'rating': self._safe_attr(item, '.rating', 'data-rating'),\n 'image': self._resolve(self._safe_attr(item, 'img', 'src')),\n 'url': self._resolve(self._safe_attr(item, 'a', 'href')),\n 'in_stock': not item.select_one('.out-of-stock')\n }\n\n def scrape_products(self, url):\n \"\"\"Scrape all products from a page.\"\"\"\n soup = self.fetch_page(url)\n items = soup.select('.product-card')\n return [self.extract_product(item) for item in items]\n\n def _safe_text(self, element, selector, default=''):\n found = element.select_one(selector)\n return found.get_text(strip=True) if found else default\n\n def _safe_attr(self, element, selector, attr, default=None):\n found = element.select_one(selector)\n return found.get(attr, default) if found else default\n\n def _parse_price(self, element):\n if not element:\n return None\n text = element.get_text(strip=True)\n try:\n return float(text.replace('

Important: agents should read /llm.txt, /llms.txt, or /.well-known/skills.json to discover the public Skillopedia API.