Overview

Link and email extraction is one of the most common web scraping tasks. Whether you're building a crawler, finding contact information, or analyzing website structure, this tutorial will show you multiple approaches.

What You'll Learn

Extract all page links

Find email addresses

Filter by domain

Get social media links

Find mailto links

Build link profiles

Use Cases

Lead generation, competitive analysis, SEO audits, contact discovery, link building, website mapping, and dead link checking.

2

Extract All Links

The simplest approach is to extract all <a> tags and their href attributes:

JSON - Extract Rules

{
  "all_links": {
    "selector": "a[href]",
    "type": "obj",
    "multiple": true,
    "children": {
      "text": { "selector": "", "type": "text" },
      "url": { "selector": "", "type": "link" },
      "title": { "selector": "", "type": "attr", "attribute": "title" }
    }
  }
}

API Request

import requests

extract_rules = {
    "all_links": {
        "selector": "a[href]",
        "type": "obj",
        "multiple": True,
        "children": {
            "text": {"selector": "", "type": "text"},
            "url": {"selector": "", "type": "link"}
        }
    }
}

response = requests.post("https://api.ujeebu.com/scrape",
    headers={
        "ApiKey": "YOUR_API_KEY",
        "Content-Type": "application/json"
    },
    json={
        "url": "https://example.com",
        "extract_rules": extract_rules
    })

links = response.json()["result"]["all_links"]

# Categorize links
internal = [l for l in links if "example.com" in l.get("url", "")]
external = [l for l in links if "example.com" not in l.get("url", "")]

print(f"Internal links: {len(internal)}")
print(f"External links: {len(external)}")

const axios = require('axios');

const extractRules = {
  all_links: {
    selector: 'a[href]',
    type: 'obj',
    multiple: true,
    children: {
      text: { selector: '', type: 'text' },
      url: { selector: '', type: 'link' }
    }
  }
};

const response = await axios.post('https://api.ujeebu.com/scrape', {
  url: 'https://example.com',
  extract_rules: extractRules
}, {
  headers: { 'ApiKey': 'YOUR_API_KEY' }
});

const links = response.data.result.all_links;
const internal = links.filter(l => l.url?.includes('example.com'));
const external = links.filter(l => !l.url?.includes('example.com'));

console.log(`Internal links: ${internal.length}`);
console.log(`External links: ${external.length}`);

curl -X POST "https://api.ujeebu.com/scrape" \
  -H "ApiKey: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url":"https://example.com","extract_rules":{"all_links":{"selector":"a[href]","type":"obj","multiple":true,"children":{"text":{"selector":"","type":"text"},"url":{"selector":"","type":"link"}}}}}'

from ujeebu_python import UjeebuClient

uj = UjeebuClient(api_key="YOUR_API_KEY")

res = uj.scrape_with_rules(
    url="https://example.com",
    extract_rules={
        "all_links": {
            "selector": "a[href]",
            "type": "obj",
            "multiple": True,
            "children": {
                "text": {
                    "selector": "",
                    "type": "text"
                },
                "url": {
                    "selector": "",
                    "type": "link"
                }
            }
        }
    },
)
print(res.json())

const { UjeebuClient } = require('@ujeebu-org/ujeebu-sdk');

const client = new UjeebuClient("YOUR_API_KEY");

(async () => {
  const res = await client.scrape("https://example.com", {
  "extract_rules": {"all_links":{"selector":"a[href]","type":"obj","multiple":true,"children":{"text":{"selector":"","type":"text"},"url":{"selector":"","type":"link"}}}},
});
  console.log(res.data);
})();

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.ujeebu.com/scrape');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_HTTPHEADER, [
    'ApiKey: ' . 'YOUR_API_KEY',
    'Content-Type: application/json',
]);
$payload = [
    'url' => 'https://example.com',
    'extract_rules' => [
        'all_links' => [
            'selector' => 'a[href]',
            'type' => 'obj',
            'multiple' => true,
            'children' => [
                'text' => [
                    'selector' => '',
                    'type' => 'text'
                ],
                'url' => [
                    'selector' => '',
                    'type' => 'link'
                ]
            ]
        ]
    ]
];
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
$response = curl_exec($ch);
curl_close($ch);
echo $response;

package main

import (
    "fmt"
    "io"
    "strings"
    "net/http"
)

func main() {
    url := "https://api.ujeebu.com/scrape"
    payload := strings.NewReader(`{"url":"https://example.com","extract_rules":{"all_links":{"selector":"a[href]","type":"obj","multiple":true,"children":{"text":{"selector":"","type":"text"},"url":{"selector":"","type":"link"}}}}}`)
    req, _ := http.NewRequest("POST", url, payload)
    req.Header.Set("ApiKey", "YOUR_API_KEY")
    req.Header.Set("Content-Type", "application/json")
    res, err := http.DefaultClient.Do(req)
    if err != nil { panic(err) }
    defer res.Body.Close()
    body, _ := io.ReadAll(res.Body)
    fmt.Println(string(body))
}

package main

import (
    "fmt"
    "log"

    "github.com/ujeebu/ujeebu-go"
)

func main() {
    client, err := ujeebu.NewClient("YOUR_API_KEY")
    if err != nil {
        log.Fatal(err)
    }

    res, _, err := client.Scrape(ujeebu.ScrapeParams{
        URL: "https://example.com",
        ExtractRules: map[string]any{
            "all_links": map[string]any{
                "selector": "a[href]",
                "type": "obj",
                "multiple": true,
                "children": map[string]any{
                    "text": map[string]any{
                        "selector": "",
                        "type": "text",
                    },
                    "url": map[string]any{
                        "selector": "",
                        "type": "link",
                    },
                },
            },
        },
    })
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(res)
}

3

Extract Email Addresses

Use mailto links and regex patterns to find email addresses:

JSON - Extract Rules

{
  "mailto_links": {
    "selector": "a[href^='mailto:']",
    "type": "obj",
    "multiple": true,
    "children": {
      "email": { "selector": "", "type": "attr", "attribute": "href" },
      "text": { "selector": "", "type": "text" }
    }
  },
  "page_content": {
    "selector": "body",
    "type": "text"
  }
}

Parse Emails with Regex

import re
import requests

EXTRACT_RULES = {
    "mailto_links": {
        "selector": "a[href^='mailto:']",
        "type": "obj",
        "multiple": True,
        "children": {
            "email": {"selector": "", "type": "attr", "attribute": "href"}
        }
    },
    "page_content": {"selector": "body", "type": "text"}
}

response = requests.post("https://api.ujeebu.com/scrape",
    headers={"ApiKey": "YOUR_API_KEY", "Content-Type": "application/json"},
    json={
        "url": "https://example.com/contact",
        "extract_rules": EXTRACT_RULES
    })

data = response.json()["result"]

# Regex for email addresses
email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'

# From mailto links
mailto_emails = [
    link["email"].replace("mailto:", "").split("?")[0]
    for link in data.get("mailto_links", [])
]

# From page content using regex
content_emails = re.findall(email_pattern, data.get("page_content", ""))

# Combine and deduplicate
all_emails = list(set(mailto_emails + content_emails))
print(f"Found {len(all_emails)} unique emails: {all_emails}")

const axios = require('axios');

const EXTRACT_RULES = {
  mailto_links: {
    selector: "a[href^='mailto:']",
    type: 'obj',
    multiple: true,
    children: {
      email: { selector: '', type: 'attr', attribute: 'href' }
    }
  },
  page_content: { selector: 'body', type: 'text' }
};

(async () => {
  const response = await axios.post('https://api.ujeebu.com/scrape', {
    url: 'https://example.com/contact',
    extract_rules: EXTRACT_RULES
  }, {
    headers: { 'ApiKey': 'YOUR_API_KEY', 'Content-Type': 'application/json' }
  });

  const data = response.data.result;

  // Regex for email addresses
  const emailPattern = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;

  // From mailto links
  const mailtoEmails = (data.mailto_links || []).map(
    link => link.email.replace('mailto:', '').split('?')[0]
  );

  // From page content using regex
  const contentEmails = (data.page_content || '').match(emailPattern) || [];

  // Combine and deduplicate
  const allEmails = [...new Set([...mailtoEmails, ...contentEmails])];
  console.log(`Found ${allEmails.length} unique emails:`, allEmails);
})();

import re
from ujeebu_python import UjeebuClient

uj = UjeebuClient(api_key="YOUR_API_KEY")

EXTRACT_RULES = {
    "mailto_links": {
        "selector": "a[href^='mailto:']",
        "type": "obj",
        "multiple": True,
        "children": {
            "email": {"selector": "", "type": "attr", "attribute": "href"}
        }
    },
    "page_content": {"selector": "body", "type": "text"}
}

res = uj.scrape_with_rules(
    "https://example.com/contact",
    extract_rules=EXTRACT_RULES)

data = res.json()["result"]

# Regex for email addresses
email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'

# From mailto links
mailto_emails = [
    link["email"].replace("mailto:", "").split("?")[0]
    for link in data.get("mailto_links", [])
]

# From page content using regex
content_emails = re.findall(email_pattern, data.get("page_content", ""))

# Combine and deduplicate
all_emails = list(set(mailto_emails + content_emails))
print(f"Found {len(all_emails)} unique emails: {all_emails}")

const { UjeebuClient } = require('@ujeebu-org/ujeebu-sdk');

const client = new UjeebuClient("YOUR_API_KEY");

const EXTRACT_RULES = {
  mailto_links: {
    selector: "a[href^='mailto:']",
    type: 'obj',
    multiple: true,
    children: {
      email: { selector: '', type: 'attr', attribute: 'href' }
    }
  },
  page_content: { selector: 'body', type: 'text' }
};

(async () => {
  const res = await client.scrape('https://example.com/contact', {
    extract_rules: EXTRACT_RULES
  });

  const data = res.data.result;

  // Regex for email addresses
  const emailPattern = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;

  // From mailto links
  const mailtoEmails = (data.mailto_links || []).map(
    link => link.email.replace('mailto:', '').split('?')[0]
  );

  // From page content using regex
  const contentEmails = (data.page_content || '').match(emailPattern) || [];

  // Combine and deduplicate
  const allEmails = [...new Set([...mailtoEmails, ...contentEmails])];
  console.log(`Found ${allEmails.length} unique emails:`, allEmails);
})();

<?php
$extractRules = [
    'mailto_links' => [
        'selector' => "a[href^='mailto:']",
        'type' => 'obj',
        'multiple' => true,
        'children' => [
            'email' => ['selector' => '', 'type' => 'attr', 'attribute' => 'href'],
        ],
    ],
    'page_content' => ['selector' => 'body', 'type' => 'text'],
];

$ch = curl_init();
curl_setopt_array($ch, [
    CURLOPT_URL => 'https://api.ujeebu.com/scrape',
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_POST => true,
    CURLOPT_HTTPHEADER => ['ApiKey: YOUR_API_KEY', 'Content-Type: application/json'],
    CURLOPT_POSTFIELDS => json_encode([
        'url' => 'https://example.com/contact',
        'extract_rules' => $extractRules,
    ]),
]);
$response = curl_exec($ch);
curl_close($ch);

$data = json_decode($response, true)['result'] ?? [];

// Regex for email addresses
$emailPattern = '/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/';

// From mailto links
$mailtoEmails = [];
foreach ($data['mailto_links'] ?? [] as $link) {
    $email = str_replace('mailto:', '', $link['email'] ?? '');
    $mailtoEmails[] = explode('?', $email)[0];
}

// From page content using regex
preg_match_all($emailPattern, $data['page_content'] ?? '', $matches);
$contentEmails = $matches[0];

// Combine and deduplicate
$allEmails = array_values(array_unique(array_merge($mailtoEmails, $contentEmails)));
echo "Found " . count($allEmails) . " unique emails: " . implode(', ', $allEmails) . "\n";

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
)

func main() {
	rules := map[string]any{
		"mailto_links": map[string]any{
			"selector": "a[href^='mailto:']",
			"type":     "obj",
			"multiple": true,
			"children": map[string]any{
				"email": map[string]any{"selector": "", "type": "attr", "attribute": "href"},
			},
		},
		"page_content": map[string]any{"selector": "body", "type": "text"},
	}
	payload, _ := json.Marshal(map[string]any{
		"url":           "https://example.com/contact",
		"extract_rules": rules,
	})
	req, _ := http.NewRequest("POST", "https://api.ujeebu.com/scrape", bytes.NewReader(payload))
	req.Header.Set("ApiKey", "YOUR_API_KEY")
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	body, _ := io.ReadAll(res.Body)
	res.Body.Close()

	var parsed struct {
		Result struct {
			MailtoLinks []struct {
				Email string `json:"email"`
			} `json:"mailto_links"`
			PageContent string `json:"page_content"`
		} `json:"result"`
	}
	json.Unmarshal(body, &parsed)

	// Regex for email addresses
	emailPattern := regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`)

	emailSet := map[string]bool{}
	// From mailto links
	for _, link := range parsed.Result.MailtoLinks {
		email := strings.SplitN(strings.Replace(link.Email, "mailto:", "", 1), "?", 2)[0]
		emailSet[email] = true
	}
	// From page content using regex
	for _, email := range emailPattern.FindAllString(parsed.Result.PageContent, -1) {
		emailSet[email] = true
	}

	// Combine and deduplicate
	var allEmails []string
	for email := range emailSet {
		allEmails = append(allEmails, email)
	}
	fmt.Printf("Found %d unique emails: %v\n", len(allEmails), allEmails)
}

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"regexp"
	"strings"

	"github.com/ujeebu/ujeebu-go"
)

func main() {
	client, err := ujeebu.NewClient("YOUR_API_KEY")
	if err != nil {
		log.Fatal(err)
	}

	rules := map[string]any{
		"mailto_links": map[string]any{
			"selector": "a[href^='mailto:']",
			"type":     "obj",
			"multiple": true,
			"children": map[string]any{
				"email": map[string]any{"selector": "", "type": "attr", "attribute": "href"},
			},
		},
		"page_content": map[string]any{"selector": "body", "type": "text"},
	}

	resp, _, err := client.Scrape(ujeebu.ScrapeParams{
		URL:          "https://example.com/contact",
		ExtractRules: rules,
	})
	if err != nil {
		log.Fatal(err)
	}

	var data struct {
		MailtoLinks []struct {
			Email string `json:"email"`
		} `json:"mailto_links"`
		PageContent string `json:"page_content"`
	}
	b, _ := json.Marshal(resp.Result)
	json.Unmarshal(b, &data)

	// Regex for email addresses
	emailPattern := regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`)

	emailSet := map[string]bool{}
	// From mailto links
	for _, link := range data.MailtoLinks {
		email := strings.SplitN(strings.Replace(link.Email, "mailto:", "", 1), "?", 2)[0]
		emailSet[email] = true
	}
	// From page content using regex
	for _, email := range emailPattern.FindAllString(data.PageContent, -1) {
		emailSet[email] = true
	}

	// Combine and deduplicate
	var allEmails []string
	for email := range emailSet {
		allEmails = append(allEmails, email)
	}
	fmt.Printf("Found %d unique emails: %v\n", len(allEmails), allEmails)
}

4

Filter Links by Type

Use CSS selectors to target specific link types:

Link Type	CSS Selector
External links	`a[href^='http']:not([href*='yourdomain.com'])`
PDF files	`a[href$='.pdf']`
Images	`a[href$='.jpg'], a[href$='.png']`
Telephone	`a[href^='tel:']`
Navigation	`nav a, header a`
Footer links	`footer a`

5

Extract Social Media Links

JSON - Extract Rules

{
  "twitter": {
    "selector": "a[href*='twitter.com'], a[href*='x.com']",
    "type": "link",
    "multiple": true
  },
  "linkedin": {
    "selector": "a[href*='linkedin.com']",
    "type": "link",
    "multiple": true
  },
  "facebook": {
    "selector": "a[href*='facebook.com']",
    "type": "link",
    "multiple": true
  },
  "instagram": {
    "selector": "a[href*='instagram.com']",
    "type": "link",
    "multiple": true
  },
  "github": {
    "selector": "a[href*='github.com']",
    "type": "link",
    "multiple": true
  }
}

6

Best Practices

01

Deduplicate Results

Pages often have duplicate links. Always remove duplicates using set() or similar.

Essential

02

Validate Emails

Use proper email validation. Regex can match invalid patterns like image@2x.png.

Recommended

03

Resolve Relative URLs

Convert relative URLs to absolute URLs using the page's base URL for proper link analysis.

Important

04

Respect Privacy

Don't spam extracted emails. Respect GDPR and anti-spam regulations when using contact data.

Legal

Extracting Links & Emails

Overview

What You'll Learn

Extract All Links

API Request

Extract Email Addresses

Parse Emails with Regex

Filter Links by Type

Best Practices

Deduplicate Results

Validate Emails

Resolve Relative URLs

Respect Privacy

Ready to Start Scraping?