For Developers

./scrapai spiders import my_config.json --project myproject

You (plain English) → AI Agent → JSON config → Database → Scrapy crawl
                       (once)                               (forever)

{
  "name": "bbc_co_uk",
  "allowed_domains": ["bbc.co.uk"],
  "start_urls": ["https://www.bbc.co.uk/news"],
  "rules": [
    {
      "allow": ["/news/articles/[^/]+$"],
      "callback": "parse_article",
      "follow": false
    },
    {
      "allow": ["/news/?$"],
      "follow": true
    }
  ],
  "settings": {
    "EXTRACTOR_ORDER": ["newspaper", "trafilatura"],
    "DOWNLOAD_DELAY": 2
  }
}

{
  "callbacks": {
    "parse_product": {
      "extract": {
        "title": {"css": "h1.product-name::text"},
        "price": {
          "css": "span.price::text",
          "processors": [
            {"type": "strip"},
            {"type": "regex", "pattern": "\\$([\\d.]+)"},
            {"type": "cast", "to": "float"}
          ]
        },
        "rating": {
          "css": "div.stars::attr(data-rating)",
          "processors": [{"type": "cast", "to": "float"}]
        }
      }
    }
  }
}

CREATE TABLE spiders (
    id INTEGER PRIMARY KEY,
    name VARCHAR(255) UNIQUE NOT NULL,
    project VARCHAR(255) NOT NULL,
    allowed_domains JSON NOT NULL,
    start_urls JSON NOT NULL,
    created_at TIMESTAMP,
    updated_at TIMESTAMP
);

CREATE TABLE spider_rules (
    id INTEGER PRIMARY KEY,
    spider_id INTEGER REFERENCES spiders(id),
    allow_patterns JSON,
    deny_patterns JSON,
    callback VARCHAR(255),
    follow BOOLEAN,
    priority INTEGER
);

CREATE TABLE spider_settings (
    id INTEGER PRIMARY KEY,
    spider_id INTEGER REFERENCES spiders(id),
    key VARCHAR(255) NOT NULL,
    value TEXT NOT NULL
);

CREATE TABLE scraped_items (
    id INTEGER PRIMARY KEY,
    spider_name VARCHAR(255),
    project VARCHAR(255),
    url TEXT,
    title TEXT,
    content TEXT,
    author VARCHAR(255),
    published_date TIMESTAMP,
    metadata_json JSON,
    scraped_at TIMESTAMP
);

from core.extractors import BaseExtractor

class MyCustomExtractor(BaseExtractor):
    def extract(self, response):
        return {
            'title': response.css('h1::text').get(),
            'content': response.css('article::text').getall(),
            'author': response.css('.author::text').get(),
            'published_date': None
        }

{
  "settings": {
    "EXTRACTOR_ORDER": ["my_custom", "newspaper", "trafilatura"]
  }
}

class MyCustomMiddleware:
    def process_request(self, request, spider):
        # Modify request before it's sent
        return None

    def process_response(self, request, response, spider):
        # Modify response after it's received
        return response

DOWNLOADER_MIDDLEWARES = {
    'middlewares.MyCustomMiddleware': 350,
}

# cli/mycommand.py
import click

@click.command()
@click.argument('spider_name')
@click.option('--project', required=True)
def mycommand(spider_name, project):
    """My custom command"""
    click.echo(f"Running command for {spider_name}")

from cli.mycommand import mycommand

cli.add_command(mycommand)

./scrapai crawl myspider --project news --limit 10
./scrapai show myspider --project news

./scrapai crawl myspider --project news
# Creates: data/news/myspider/2026-03-01_143022.jsonl

You: "Migrate my spider at scripts/bbc_spider.py to ScrapAI"
Agent: [Reads Python, extracts URL patterns and selectors, writes JSON config, tests, saves to database]

git clone https://github.com/discourselab/scrapai-cli.git
cd scrapai-cli
./scrapai setup
./scrapai verify

# Run all tests
pytest

# Run specific test file
pytest tests/test_extractors.py

# Run with coverage
pytest --cov=core --cov=cli --cov-report=html

# Format code
black .

# Check linting
flake8 core/ cli/ spiders/