import requests
import time

EXTRACT_RULES = {
    "jobs": {
        "selector": ".job-card",
        "type": "obj",
        "multiple": True,
        "children": {
            "title": {"selector": "h2, h3", "type": "text"},
            "company": {"selector": ".company-name", "type": "text"},
            "location": {"selector": ".location", "type": "text"},
            "salary": {"selector": ".salary", "type": "text"},
            "url": {"selector": "a", "type": "link"}
        }
    },
    "next_page": {"selector": "a[rel='next']", "type": "link"}
}

def scrape_jobs(url):
    """Scrape a single page of job listings."""
    response = requests.post("https://api.ujeebu.com/scrape",
        headers={"ApiKey": "YOUR_API_KEY", "Content-Type": "application/json"},
        json={
            "url": url,
            "js": True,
            "wait_for": ".job-card",
            "extract_rules": EXTRACT_RULES
        })
    return response.json().get("result", {})

def scrape_all_jobs(base_url, max_pages=10):
    """Scrape jobs across multiple pages."""
    all_jobs = []
    current_url = base_url
    page = 1

    while current_url and page <= max_pages:
        print(f"Scraping page {page}...")
        data = scrape_jobs(current_url)

        jobs = data.get('jobs', [])
        if not jobs:
            break

        all_jobs.extend(jobs)

        # Get next page URL
        next_page = data.get('next_page')
        if next_page:
            current_url = next_page if next_page.startswith('http') else base_url + next_page
        else:
            break

        page += 1
        time.sleep(2)  # Rate limiting

    return all_jobs

# Scrape up to 10 pages
all_jobs = scrape_all_jobs("https://example-jobboard.com/search?q=developer", max_pages=10)
print(f"Total jobs collected: {len(all_jobs)}")