Site Crawler Skill

Respectfully crawl documentation sites and web content for RAG ingestion.

Overview

Documentation sites, blogs, and knowledge bases contain valuable structured content. This skill covers:

• Respectful crawling (robots.txt, rate limiting)
• Structure-preserving extraction
• Incremental updates (only fetch changed pages)
• Sitemap-based discovery

Prerequisites

# HTTP client
pip install httpx

# HTML parsing
pip install beautifulsoup4 lxml

# Clean article extraction
pip install trafilatura

# Markdown conversion
pip install markdownify

Crawling Principles

1. Be Respectful

• Always check robots.txt
• Rate limit requests (1-2 seconds between)
• Identify yourself with a User-Agent
• Don't overload servers

2. Be Efficient

• Use sitemaps when available
• Track what's been crawled
• Only re-fetch changed content
• Skip non-content pages (login, search results)

3. Be Smart

• Preserve document structure
• Extract meaningful content only
• Handle pagination
• Detect and follow documentation structure

Core Implementation

Robots.txt Handling

#!/usr/bin/env python3
"""Robots.txt compliance."""

from urllib.robotparser import RobotFileParser
from urllib.parse import urljoin, urlparse
from typing import Optional
import httpx

class RobotsChecker:
    """Check robots.txt compliance before crawling."""

    def __init__(self, user_agent: str = "ContentHarvester/1.0"):
        self.user_agent = user_agent
        self.parsers: dict = {}

    async def can_fetch(self, url: str) -> bool:
        """Check if URL can be fetched according to robots.txt."""
        parsed = urlparse(url)
        base_url = f"{parsed.scheme}://{parsed.netloc}"

        if base_url not in self.parsers:
            await self._load_robots(base_url)

        parser = self.parsers.get(base_url)
        if parser is None:
            return True  # No robots.txt = allow all

        return parser.can_fetch(self.user_agent, url)

    async def _load_robots(self, base_url: str):
        """Load and parse robots.txt."""
        robots_url = f"{base_url}/robots.txt"

        try:
            async with httpx.AsyncClient() as client:
                response = await client.get(robots_url, timeout=10)

            if response.status_code == 200:
                parser = RobotFileParser()
                parser.parse(response.text.split("
"))
                self.parsers[base_url] = parser
            else:
                self.parsers[base_url] = None

        except Exception:
            self.parsers[base_url] = None

    def get_crawl_delay(self, base_url: str) -> Optional[float]:
        """Get crawl delay from robots.txt."""
        parser = self.parsers.get(base_url)
        if parser:
            delay = parser.crawl_delay(self.user_agent)
            return delay if delay else None
        return None

Content Extractor

#!/usr/bin/env python3
"""Clean content extraction from HTML."""

from bs4 import BeautifulSoup
import trafilatura
from markdownify import markdownify as md
from typing import Dict, Optional
import re

def extract_content(html: str, url: str) -> Dict:
    """
    Extract clean content from HTML.

    Uses multiple strategies for best results.
    """
    result = {
        "title": "",
        "content": "",
        "markdown": "",
        "headings": [],
        "links": [],
        "metadata": {}
    }

    soup = BeautifulSoup(html, 'lxml')

    # Get title
    title_tag = soup.find('title')
    if title_tag:
        result["title"] = title_tag.get_text().strip()

    # Try trafilatura for clean extraction
    extracted = trafilatura.extract(
        html,
        include_comments=False,
        include_tables=True,
        include_links=True,
        output_format='markdown'
    )

    if extracted:
        result["markdown"] = extracted
        result["content"] = trafilatura.extract(html, output_format='txt') or ""
    else:
        # Fallback to manual extraction
        result["markdown"] = extract_main_content(soup)
        result["content"] = soup.get_text(separator=' ', strip=True)

    # Extract headings for structure
    for heading in soup.find_all(['h1', 'h2', 'h3', 'h4']):
        result["headings"].append({
            "level": int(heading.name[1]),
            "text": heading.get_text().strip()
        })

    # Extract metadata
    for meta in soup.find_all('meta'):
        name = meta.get('name', meta.get('property', ''))
        content = meta.get('content',