Projects Commands

./scrapai projects list

$ ./scrapai projects list
📁 Available Projects:
  • news
    Spiders: 8, Queue items: 23
  • ecommerce
    Spiders: 12, Queue items: 47
  • research
    Spiders: 5, Queue items: 0
  • archive
    Spiders: 3, Queue items: 2

$ ./scrapai projects list
No projects found.

# Import spider creates project "news"
./scrapai spiders import bbc_spider.json --project news

# Add queue item creates project "research"
./scrapai queue add https://example.com --project research

# News aggregation project
./scrapai spiders import bbc.json --project news
./scrapai spiders import cnn.json --project news
./scrapai spiders import reuters.json --project news

# E-commerce monitoring project
./scrapai spiders import amazon.json --project ecommerce
./scrapai spiders import ebay.json --project ecommerce

# Marketing team
./scrapai spiders import competitors.json --project marketing

# Research team
./scrapai spiders import papers.json --project research

# Sales team
./scrapai spiders import leads.json --project sales

# Development/testing
./scrapai spiders import test_spider.json --project dev
./scrapai crawl test_spider --project dev --limit 5

# Production
./scrapai spiders import prod_spider.json --project prod
./scrapai crawl prod_spider --project prod

./scrapai spiders list --project news

./scrapai crawl-all --project news

# Add to project queue
./scrapai queue add https://example.com --project news

# List project queue
./scrapai queue list --project news

# Process project queue
./scrapai queue next --project news

# Export all spiders in project
for spider in $(./scrapai spiders list --project news | grep '•' | awk '{print $2}'); do
  ./scrapai export $spider --project news --format csv
done

data/
├── news/
│   ├── bbc_co_uk/
│   │   ├── crawls/
│   │   ├── exports/
│   │   └── checkpoint/
│   ├── cnn_com/
│   └── reuters_com/
├── ecommerce/
│   ├── amazon_spider/
│   └── ebay_spider/
└── research/
    └── papers_spider/

CREATE TABLE spiders (
    id INTEGER PRIMARY KEY,
    name VARCHAR(255) NOT NULL,
    project VARCHAR(255),  -- Project name
    ...
);

CREATE TABLE crawl_queue (
    id INTEGER PRIMARY KEY,
    project_name VARCHAR(255) NOT NULL,  -- Project name
    website_url VARCHAR(2048) NOT NULL,
    ...
);

# Rename project "old_name" to "new_name"
./scrapai db query "UPDATE spiders SET project='new_name' WHERE project='old_name'" --yes
./scrapai db query "UPDATE crawl_queue SET project_name='new_name' WHERE project_name='old_name'" --yes

# List spiders first
./scrapai spiders list --project old_project

# Delete each spider
./scrapai spiders delete spider1 --project old_project --force
./scrapai spiders delete spider2 --project old_project --force

./scrapai queue cleanup --project old_project --all --force

rm -rf data/old_project

# Terminal 1: News project
while true; do
  ./scrapai queue next --project news && # process
done

# Terminal 2: E-commerce project
while true; do
  ./scrapai queue next --project ecommerce && # process
done

# Terminal 3: Research project
while true; do
  ./scrapai queue next --project research && # process
done

# Spider count
./scrapai db query "SELECT project, COUNT(*) FROM spiders GROUP BY project"

# Item count per project
./scrapai db query "
SELECT s.project, COUNT(si.id) as items
FROM spiders s
LEFT JOIN scraped_items si ON s.id = si.spider_id
GROUP BY s.project
ORDER BY items DESC
"

# Queue status by project
./scrapai db query "
SELECT project_name, status, COUNT(*) 
FROM crawl_queue 
GROUP BY project_name, status
"

# Update project in database
./scrapai db query "UPDATE spiders SET project='new_project' WHERE project='old_project'" --yes

# Move data directory
mv data/old_project data/new_project