Extracting Links & Emails

{
  "all_links": {
    "selector": "a[href]",
    "type": "obj",
    "multiple": true,
    "children": {
      "text": { "selector": "", "type": "text" },
      "url": { "selector": "", "type": "link" },
      "title": { "selector": "", "type": "attr", "attribute": "title" }
    }
  }
}

import requests

extract_rules = {
    "all_links": {
        "selector": "a[href]",
        "type": "obj",
        "multiple": True,
        "children": {
            "text": {"selector": "", "type": "text"},
            "url": {"selector": "", "type": "link"}
        }
    }
}

response = requests.post("https://api.ujeebu.com/scrape",
    headers={
        "ApiKey": "YOUR_API_KEY",
        "Content-Type": "application/json"
    },
    json={
        "url": "https://example.com",
        "extract_rules": extract_rules
    })

links = response.json()["result"]["all_links"]

# Categorize links
internal = [l for l in links if "example.com" in l.get("url", "")]
external = [l for l in links if "example.com" not in l.get("url", "")]

print(f"Internal links: {len(internal)}")
print(f"External links: {len(external)}")

const axios = require('axios');

const extractRules = {
  all_links: {
    selector: 'a[href]',
    type: 'obj',
    multiple: true,
    children: {
      text: { selector: '', type: 'text' },
      url: { selector: '', type: 'link' }
    }
  }
};

const response = await axios.post('https://api.ujeebu.com/scrape', {
  url: 'https://example.com',
  extract_rules: extractRules
}, {
  headers: { 'ApiKey': 'YOUR_API_KEY' }
});

const links = response.data.result.all_links;
const internal = links.filter(l => l.url?.includes('example.com'));
const external = links.filter(l => !l.url?.includes('example.com'));

console.log(`Internal links: ${internal.length}`);
console.log(`External links: ${external.length}`);

{
  "mailto_links": {
    "selector": "a[href^='mailto:']",
    "type": "obj",
    "multiple": true,
    "children": {
      "email": { "selector": "", "type": "attr", "attribute": "href" },
      "text": { "selector": "", "type": "text" }
    }
  },
  "page_content": {
    "selector": "body",
    "type": "text"
  }
}

import re

# Extract emails from page content
email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'

data = response.json()["result"]

# From mailto links
mailto_emails = [
    link["email"].replace("mailto:", "").split("?")[0]
    for link in data.get("mailto_links", [])
]

# From page content using regex
content_emails = re.findall(email_pattern, data.get("page_content", ""))

# Combine and deduplicate
all_emails = list(set(mailto_emails + content_emails))
print(f"Found {len(all_emails)} unique emails: {all_emails}")

{
  "twitter": {
    "selector": "a[href*='twitter.com'], a[href*='x.com']",
    "type": "link",
    "multiple": true
  },
  "linkedin": {
    "selector": "a[href*='linkedin.com']",
    "type": "link",
    "multiple": true
  },
  "facebook": {
    "selector": "a[href*='facebook.com']",
    "type": "link",
    "multiple": true
  },
  "instagram": {
    "selector": "a[href*='instagram.com']",
    "type": "link",
    "multiple": true
  },
  "github": {
    "selector": "a[href*='github.com']",
    "type": "link",
    "multiple": true
  }
}