Scraping Hacker News Feed

{
  "stories": {
    "selector": "tr.athing",
    "type": "obj",
    "multiple": true,
    "children": {
      "rank": { "selector": ".rank", "type": "text" },
      "title": { "selector": ".titleline > a", "type": "text" },
      "url": { "selector": ".titleline > a", "type": "link" },
      "domain": { "selector": ".sitestr", "type": "text" },
      "story_id": { "selector": "", "type": "attr", "attribute": "id" },
      "points": {
        "type": "fn",
        "fn": "return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"
      },
      "author": {
        "type": "fn",
        "fn": "return $parent.nextElementSibling?.querySelector('.hnuser')?.textContent || null;"
      },
      "time": {
        "type": "fn",
        "fn": "return $parent.nextElementSibling?.querySelector('.age')?.textContent || null;"
      },
      "comments_text": {
        "type": "fn",
        "fn": "const links = Array.from($parent.nextElementSibling?.querySelectorAll('.subtext a') || []); return links.length ? links[links.length-1].textContent : null;"
      }
    }
  }
}

curl -X POST "https://api.ujeebu.com/scrape" \
  -H "ApiKey: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://news.ycombinator.com/",
    "js": true,
    "extract_rules": {
      "stories": {
        "selector": "tr.athing",
        "type": "obj",
        "multiple": true,
        "children": {
          "title": {"selector": ".titleline > a", "type": "text"},
          "url": {"selector": ".titleline > a", "type": "link"},
          "points": {"type": "fn", "fn": "return $parent.nextElementSibling?.querySelector('"'"'.score'"'"')?.textContent || null;"}
        }
      }
    }
  }'

import requests

extract_rules = {
    "stories": {
        "selector": "tr.athing",
        "type": "obj",
        "multiple": True,
        "children": {
            "rank": {"selector": ".rank", "type": "text"},
            "title": {"selector": ".titleline > a", "type": "text"},
            "url": {"selector": ".titleline > a", "type": "link"},
            "domain": {"selector": ".sitestr", "type": "text"},
            "points": {
                "type": "fn",
                "fn": "return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"
            },
            "author": {
                "type": "fn",
                "fn": "return $parent.nextElementSibling?.querySelector('.hnuser')?.textContent || null;"
            },
            "time": {
                "type": "fn",
                "fn": "return $parent.nextElementSibling?.querySelector('.age')?.textContent || null;"
            },
            "comments_text": {
                "type": "fn",
                "fn": "const links = Array.from($parent.nextElementSibling?.querySelectorAll('.subtext a') || []); return links.length ? links[links.length-1].textContent : null;"
            }
        }
    }
}

response = requests.post("https://api.ujeebu.com/scrape",
    headers={
        "ApiKey": "YOUR_API_KEY",
        "Content-Type": "application/json"
    },
    json={
        "url": "https://news.ycombinator.com/",
        "js": True,
        "extract_rules": extract_rules
    })

stories = response.json()["result"]["stories"]
for story in stories[:5]:
    print(f"{story['rank']} {story['title']}")
    print(f"   {story.get('points', 'N/A')} by {story.get('author', 'N/A')}")

const axios = require('axios');

const extractRules = {
  stories: {
    selector: 'tr.athing',
    type: 'obj',
    multiple: true,
    children: {
      rank: { selector: '.rank', type: 'text' },
      title: { selector: '.titleline > a', type: 'text' },
      url: { selector: '.titleline > a', type: 'link' },
      domain: { selector: '.sitestr', type: 'text' },
      points: {
        type: 'fn',
        fn: "return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"
      },
      author: {
        type: 'fn',
        fn: "return $parent.nextElementSibling?.querySelector('.hnuser')?.textContent || null;"
      }
    }
  }
};

const response = await axios.post('https://api.ujeebu.com/scrape', {
  url: 'https://news.ycombinator.com/',
  js: true,
  extract_rules: extractRules
}, {
  headers: { 'ApiKey': 'YOUR_API_KEY' }
});

const stories = response.data.result.stories;
stories.slice(0, 5).forEach(story => {
  console.log(`${story.rank} ${story.title}`);
  console.log(`   ${story.points || 'N/A'} by ${story.author || 'N/A'}`);
});

from ujeebu_python import UjeebuClient

uj = UjeebuClient(api_key="YOUR_API_KEY")

res = uj.scrape_with_rules(
    url="https://news.ycombinator.com/",
    extract_rules={
        "stories": {
            "selector": "tr.athing",
            "type": "obj",
            "multiple": True,
            "children": {
                "title": {
                    "selector": ".titleline > a",
                    "type": "text"
                },
                "url": {
                    "selector": ".titleline > a",
                    "type": "link"
                },
                "points": {
                    "type": "fn",
                    "fn": "return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"
                }
            }
        }
    },
    params={
        "js": True
    },
)
print(res.json())

const { UjeebuClient } = require('@ujeebu-org/ujeebu-sdk');

const client = new UjeebuClient("YOUR_API_KEY");

(async () => {
  const res = await client.scrape("https://news.ycombinator.com/", {
  "js": true,
  "extract_rules": {"stories":{"selector":"tr.athing","type":"obj","multiple":true,"children":{"title":{"selector":".titleline > a","type":"text"},"url":{"selector":".titleline > a","type":"link"},"points":{"type":"fn","fn":"return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"}}}},
});
  console.log(res.data);
})();

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://api.ujeebu.com/scrape');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_HTTPHEADER, [
    'ApiKey: ' . 'YOUR_API_KEY',
    'Content-Type: application/json',
]);
$payload = [
    'url' => 'https://news.ycombinator.com/',
    'js' => true,
    'extract_rules' => [
        'stories' => [
            'selector' => 'tr.athing',
            'type' => 'obj',
            'multiple' => true,
            'children' => [
                'title' => [
                    'selector' => '.titleline > a',
                    'type' => 'text'
                ],
                'url' => [
                    'selector' => '.titleline > a',
                    'type' => 'link'
                ],
                'points' => [
                    'type' => 'fn',
                    'fn' => 'return $parent.nextElementSibling?.querySelector(\'.score\')?.textContent || null;'
                ]
            ]
        ]
    ]
];
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($payload));
$response = curl_exec($ch);
curl_close($ch);
echo $response;

package main

import (
    "fmt"
    "io"
    "strings"
    "net/http"
)

func main() {
    url := "https://api.ujeebu.com/scrape"
    payload := strings.NewReader(`{"url":"https://news.ycombinator.com/","js":true,"extract_rules":{"stories":{"selector":"tr.athing","type":"obj","multiple":true,"children":{"title":{"selector":".titleline > a","type":"text"},"url":{"selector":".titleline > a","type":"link"},"points":{"type":"fn","fn":"return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;"}}}}}`)
    req, _ := http.NewRequest("POST", url, payload)
    req.Header.Set("ApiKey", "YOUR_API_KEY")
    req.Header.Set("Content-Type", "application/json")
    res, err := http.DefaultClient.Do(req)
    if err != nil { panic(err) }
    defer res.Body.Close()
    body, _ := io.ReadAll(res.Body)
    fmt.Println(string(body))
}

package main

import (
    "fmt"
    "log"

    "github.com/ujeebu/ujeebu-go"
)

func main() {
    client, err := ujeebu.NewClient("YOUR_API_KEY")
    if err != nil {
        log.Fatal(err)
    }

    res, _, err := client.Scrape(ujeebu.ScrapeParams{
        URL: "https://news.ycombinator.com/",
        ExtractRules: map[string]any{
            "stories": map[string]any{
                "selector": "tr.athing",
                "type": "obj",
                "multiple": true,
                "children": map[string]any{
                    "title": map[string]any{
                        "selector": ".titleline > a",
                        "type": "text",
                    },
                    "url": map[string]any{
                        "selector": ".titleline > a",
                        "type": "link",
                    },
                    "points": map[string]any{
                        "type": "fn",
                        "fn": "return $parent.nextElementSibling?.querySelector('.score')?.textContent || null;",
                    },
                },
            },
        },
        JS: true,
    })
    if err != nil {
        log.Fatal(err)
    }
    fmt.Println(res)
}

{
  "success": true,
  "result": {
    "stories": [
      {
        "rank": "1.",
        "title": "Show HN: I built an open-source alternative to Notion",
        "url": "https://example.com/article",
        "domain": "example.com",
        "story_id": "38774523",
        "points": "342 points",
        "author": "username",
        "time": "3 hours ago",
        "comments_text": "156 comments"
      }
    ]
  }
}