site-crawler

What this skill does

# Site Crawler Skill

> Respectfully crawl documentation sites and web content for RAG ingestion.

## Overview

Documentation sites, blogs, and knowledge bases contain valuable structured content. This skill covers:
- Respectful crawling (robots.txt, rate limiting)
- Structure-preserving extraction
- Incremental updates (only fetch changed pages)
- Sitemap-based discovery

## Prerequisites

```bash
# HTTP client
pip install httpx

# HTML parsing
pip install beautifulsoup4 lxml

# Clean article extraction
pip install trafilatura

# Markdown conversion
pip install markdownify
```

## Crawling Principles

### 1. Be Respectful
- Always check robots.txt
- Rate limit requests (1-2 seconds between)
- Identify yourself with a User-Agent
- Don't overload servers

### 2. Be Efficient
- Use sitemaps when available
- Track what's been crawled
- Only re-fetch changed content
- Skip non-content pages (login, search results)

### 3. Be Smart
- Preserve document structure
- Extract meaningful content only
- Handle pagination
- Detect and follow documentation structure

## Core Implementation

### Robots.txt Handling

```python
#!/usr/bin/env python3
"""Robots.txt compliance."""

from urllib.robotparser import RobotFileParser
from urllib.parse import urljoin, urlparse
from typing import Optional
import httpx

class RobotsChecker:
    """Check robots.txt compliance before crawling."""

    def __init__(self, user_agent: str = "ContentHarvester/1.0"):
        self.user_agent = user_agent
        self.parsers: dict = {}

    async def can_fetch(self, url: str) -> bool:
        """Check if URL can be fetched according to robots.txt."""
        parsed = urlparse(url)
        base_url = f"{parsed.scheme}://{parsed.netloc}"

        if base_url not in self.parsers:
            await self._load_robots(base_url)

        parser = self.parsers.get(base_url)
        if parser is None:
            return True  # No robots.txt = allow all

        return parser.can_fetch(self.user_agent, url)

    async def _load_robots(self, base_url: str):
        """Load and parse robots.txt."""
        robots_url = f"{base_url}/robots.txt"

        try:
            async with httpx.AsyncClient() as client:
                response = await client.get(robots_url, timeout=10)

            if response.status_code == 200:
                parser = RobotFileParser()
                parser.parse(response.text.split("
"))
                self.parsers[base_url] = parser
            else:
                self.parsers[base_url] = None

        except Exception:
            self.parsers[base_url] = None

    def get_crawl_delay(self, base_url: str) -> Optional[float]:
        """Get crawl delay from robots.txt."""
        parser = self.parsers.get(base_url)
        if parser:
            delay = parser.crawl_delay(self.user_agent)
            return delay if delay else None
        return None
```

### Content Extractor

```python
#!/usr/bin/env python3
"""Clean content extraction from HTML."""

from bs4 import BeautifulSoup
import trafilatura
from markdownify import markdownify as md
from typing import Dict, Optional
import re

def extract_content(html: str, url: str) -> Dict:
    """
    Extract clean content from HTML.

    Uses multiple strategies for best results.
    """
    result = {
        "title": "",
        "content": "",
        "markdown": "",
        "headings": [],
        "links": [],
        "metadata": {}
    }

    soup = BeautifulSoup(html, 'lxml')

    # Get title
    title_tag = soup.find('title')
    if title_tag:
        result["title"] = title_tag.get_text().strip()

    # Try trafilatura for clean extraction
    extracted = trafilatura.extract(
        html,
        include_comments=False,
        include_tables=True,
        include_links=True,
        output_format='markdown'
    )

    if extracted:
        result["markdown"] = extracted
        result["content"] = trafilatura.extract(html, output_format='txt') or ""
    else:
        # Fallback to manual extraction
        result["markdown"] = extract_main_content(soup)
        result["content"] = soup.get_text(separator=' ', strip=True)

    # Extract headings for structure
    for heading in soup.find_all(['h1', 'h2', 'h3', 'h4']):
        result["headings"].append({
            "level": int(heading.name[1]),
            "text": heading.get_text().strip()
        })

    # Extract metadata
    for meta in soup.find_all('meta'):
        name = meta.get('name', meta.get('property', ''))
        content = meta.get('content', '')
        if name and content:
            result["metadata"][name] = content

    # Extract internal links for crawling
    for link in soup.find_all('a', href=True):
        href = link['href']
        if href.startswith('/') or href.startswith(url):
            result["links"].append(href)

    return result


def extract_main_content(soup: BeautifulSoup) -> str:
    """Extract main content area, removing navigation/footer."""
    # Remove unwanted elements
    for tag in soup.find_all(['nav', 'footer', 'aside', 'script', 'style', 'header']):
        tag.decompose()

    # Try to find main content area
    main = (
        soup.find('main') or
        soup.find('article') or
        soup.find('div', class_=re.compile(r'content|main|post|article', re.I)) or
        soup.find('body')
    )

    if main:
        # Convert to markdown
        return md(str(main), heading_style="ATX", strip=['script', 'style'])

    return ""


def extract_docs_structure(html: str, url: str) -> Dict:
    """
    Extract documentation-specific structure.

    Handles common doc frameworks: Docusaurus, MkDocs, Sphinx, GitBook, etc.
    """
    soup = BeautifulSoup(html, 'lxml')

    structure = {
        "title": "",
        "breadcrumbs": [],
        "sidebar_links": [],
        "content": "",
        "prev_page": None,
        "next_page": None
    }

    # Title
    title = soup.find('h1') or soup.find('title')
    if title:
        structure["title"] = title.get_text().strip()

    # Breadcrumbs (common in docs)
    breadcrumb = soup.find(class_=re.compile(r'breadcrumb', re.I))
    if breadcrumb:
        structure["breadcrumbs"] = [
            a.get_text().strip()
            for a in breadcrumb.find_all('a')
        ]

    # Sidebar navigation
    sidebar = soup.find(class_=re.compile(r'sidebar|nav|menu', re.I))
    if sidebar:
        for link in sidebar.find_all('a', href=True):
            structure["sidebar_links"].append({
                "text": link.get_text().strip(),
                "href": link['href']
            })

    # Prev/Next navigation
    prev_link = soup.find('a', class_=re.compile(r'prev', re.I))
    next_link = soup.find('a', class_=re.compile(r'next', re.I))

    if prev_link:
        structure["prev_page"] = prev_link.get('href')
    if next_link:
        structure["next_page"] = next_link.get('href')

    # Main content
    structure["content"] = extract_main_content(soup)

    return structure
```

### Site Crawler

```python
#!/usr/bin/env python3
"""Full site crawler implementation."""

import asyncio
import httpx
from urllib.parse import urljoin, urlparse
from typing import Dict, List, Set, Optional
from datetime import datetime
import hashlib
import xml.etree.ElementTree as ET

class SiteCrawler:
    """Crawl a site respectfully and extract content."""

    def __init__(
        self,
        base_url: str,
        user_agent: str = "ContentHarvester/1.0",
        rate_limit: float = 1.0,  # seconds between requests
        max_pages: int = 100
    ):
        self.base_url = base_url.rstrip('/')
        self.domain = urlparse(base_url).netloc
        self.user_agent = user_agent
        self.rate_limit = rate_limit
        self.max_pages = max_pages

        self.robots = RobotsChecker(user_agent)
        self.visited: Set[str] = set()
        self.results: List[Dict] = []

    def _normalize_url(self, url: str) -> str:
        """Normalize URL for deduplication."""
        # Remove