Data Commands

./scrapai show <spider> --project <name> [options]

# Show last 5 items
./scrapai show bbc_co_uk --project news

# Show last 10 items
./scrapai show bbc_co_uk --project news --limit 10

# Filter by URL pattern
./scrapai show bbc_co_uk --project news --url "/technology/"

# Search titles
./scrapai show bbc_co_uk --project news --title "climate"

# Search title and content
./scrapai show bbc_co_uk --project news --text "election"

$ ./scrapai show bbc_co_uk --project news --limit 3
📰 Showing 3 articles from 'bbc_co_uk':

🔸 [1] UK economy grows 0.4% in February
   📅 Published: 2026-02-28 | Scraped: 2026-02-28 15:30
   🔗 https://bbc.co.uk/news/business-123456
   ✍️  By Economics Reporter
   📝 The UK economy grew by 0.4% in February, official figures show, 
        beating economists' expectations of 0.2% growth...

🔸 [2] NASA announces Mars mission timeline
   📅 Published: 2026-02-28 | Scraped: 2026-02-28 15:32
   🔗 https://bbc.co.uk/news/science-789012
   📝 NASA has unveiled an ambitious timeline for its first crewed mission 
        to Mars, targeting a 2035 launch date...

🔸 [3] New AI regulations announced
   📅 Published: 2026-02-27 | Scraped: 2026-02-28 15:35
   🔗 https://bbc.co.uk/news/technology-345678
   ✍️  By Tech Correspondent
   📝 The government has announced new regulations for artificial intelligence 
        systems, focusing on transparency and accountability...

$ ./scrapai show ecommerce_spider --project shop --limit 2
📰 Showing 2 articles from 'ecommerce_spider':

🔸 [1] parse_product item
   📅 Scraped: 2026-02-28 16:10
   🔗 https://shop.example.com/products/widget-pro-3000
   • title: Widget Pro 3000
   • price: $299.99
   • rating: 4.5
   • reviews_count: 1,247
   • availability: In Stock

./scrapai export <spider> --project <name> --format <fmt> [options]

# Export to CSV (default location)
./scrapai export bbc_co_uk --project news --format csv

# Export to JSON with custom path
./scrapai export bbc_co_uk --project news --format json --output my_data.json

# Export to Parquet (requires pandas)
./scrapai export bbc_co_uk --project news --format parquet

# Export filtered items
./scrapai export bbc_co_uk --project news --format jsonl --title "climate"

# Export limited items
./scrapai export bbc_co_uk --project news --format csv --limit 100

$ ./scrapai export bbc_co_uk --project news --format csv
✅ Exported 247 articles to CSV: data/news/bbc_co_uk/exports/export_28022026_153042.csv

id,url,title,content,author,published_date,scraped_at
1,https://bbc.co.uk/news/business-123456,"UK economy grows 0.4%","The UK economy...","Economics Reporter",2026-02-28,2026-02-28T15:30:42
2,https://bbc.co.uk/news/science-789012,"NASA announces Mars mission","NASA has unveiled...","",2026-02-28,2026-02-28T15:32:15

[
  {
    "id": 1,
    "url": "https://bbc.co.uk/news/business-123456",
    "title": "UK economy grows 0.4%",
    "content": "The UK economy...",
    "author": "Economics Reporter",
    "published_date": "2026-02-28",
    "scraped_at": "2026-02-28T15:30:42"
  },
  {
    "id": 2,
    "url": "https://bbc.co.uk/news/science-789012",
    "title": "NASA announces Mars mission",
    "content": "NASA has unveiled...",
    "author": null,
    "published_date": "2026-02-28",
    "scraped_at": "2026-02-28T15:32:15"
  }
]

{"id": 1, "url": "https://bbc.co.uk/news/business-123456", "title": "UK economy grows 0.4%", "scraped_at": "2026-02-28T15:30:42"}
{"id": 2, "url": "https://bbc.co.uk/news/science-789012", "title": "NASA announces Mars mission", "scraped_at": "2026-02-28T15:32:15"}

.venv/bin/pip install pandas pyarrow
./scrapai export bbc_co_uk --project news --format parquet

data/<project>/<spider>/exports/export_<timestamp>.<format>

CREATE TABLE scraped_items (
    id INTEGER PRIMARY KEY,
    spider_id INTEGER NOT NULL,
    url VARCHAR(2048) NOT NULL,
    title TEXT,
    content TEXT,
    author VARCHAR(255),
    published_date TIMESTAMP,
    scraped_at TIMESTAMP NOT NULL,
    metadata_json JSON,
    FOREIGN KEY (spider_id) REFERENCES spiders(id) ON DELETE CASCADE
);