Database-First Spider Management

project/
├── spiders/
│   ├── bbc_spider.py
│   ├── cnn_spider.py
│   ├── nytimes_spider.py
│   ├── guardian_spider.py
│   └── ... 96 more files
├── settings.py
└── scrapy.cfg

spiders
├── id (primary key)
├── name (unique, indexed)
├── allowed_domains (JSON)
├── start_urls (JSON)
├── source_url (text)
├── active (boolean)
├── project (string)
├── callbacks_config (JSON)
├── created_at (timestamp)
└── updated_at (timestamp)

spider_rules
├── id (primary key)
├── spider_id (foreign key)
├── allow_patterns (JSON)
├── deny_patterns (JSON)
├── callback (string)
├── follow (boolean)
└── priority (integer)

spider_settings
├── id (primary key)
├── spider_id (foreign key)
├── key (string)
├── value (string)
└── type (string)

./scrapai spiders list --project news

SELECT name, active, created_at, updated_at 
FROM spiders 
WHERE project = 'news'
ORDER BY created_at DESC;

from pydantic import BaseModel, Field, validator
import re

class SpiderConfig(BaseModel):
    name: str = Field(pattern=r"^[a-zA-Z0-9_-]+$")  # Strict naming
    allowed_domains: list[str] = Field(min_items=1)
    start_urls: list[str] = Field(min_items=1)
    rules: list[RuleConfig] = []
    settings: dict = {}
    
    @validator("start_urls")
    def validate_urls(cls, urls):
        for url in urls:
            if not url.startswith(("http://", "https://")):
                raise ValueError(f"Invalid URL: {url}")
            # Block private IPs
            if any(ip in url for ip in ["127.0.0.1", "localhost", "10.", "192.168."]):
                raise ValueError(f"Private IP blocked: {url}")
        return urls

Rule(
    LinkExtractor(
        allow=[r"/news/articles/.*"],
        deny=[r"/news/.*#comments"]
    ),
    callback="parse_article",
    follow=False
)

# List all spiders
./scrapai db query "SELECT name, project, active FROM spiders"

# Count items per spider
./scrapai db query "SELECT spider_id, COUNT(*) FROM scraped_items GROUP BY spider_id"

# Find spiders with Cloudflare enabled
./scrapai db query "SELECT s.name FROM spiders s JOIN spider_settings ss ON s.id = ss.spider_id WHERE ss.key = 'CLOUDFLARE_ENABLED' AND ss.value = 'true'"

# Get active spiders
./scrapai db query "SELECT name FROM spiders WHERE active = true AND project = 'news'" > active_spiders.txt

# Test each spider (5 sample URLs)
while read spider; do
  echo "Testing $spider..."
  ./scrapai crawl "$spider" --project news --limit 5
done < active_spiders.txt

# Check for extraction failures
./scrapai db query "SELECT spider_id, COUNT(*) FROM scraped_items WHERE scraped_at > NOW() - INTERVAL '1 day' GROUP BY spider_id HAVING COUNT(*) = 0"

-- Increase delay for aggressive sites
UPDATE spider_settings 
SET value = '3'
WHERE key = 'DOWNLOAD_DELAY' 
  AND spider_id IN (
    SELECT id FROM spiders 
    WHERE allowed_domains::text LIKE '%amazon%' 
       OR allowed_domains::text LIKE '%ebay%'
  );

# Export current config
./scrapai spiders export bbc_co_uk --project news > bbc_v1.json

# Make changes (via agent or manual edit)
# ... modify rules, settings, etc.

# Test new version
./scrapai crawl bbc_co_uk --project news --limit 10

# Rollback if needed
./scrapai spiders import bbc_v1.json --project news

# Create separate projects for different use cases
./scrapai spiders import news_spider.json --project production
./scrapai spiders import news_spider.json --project staging
./scrapai spiders import news_spider.json --project testing

# Each project has independent configs and data
./scrapai crawl news_spider --project production
./scrapai crawl news_spider --project testing --limit 5