Go SDK
The official Ujeebu SDK for Go. Built with idiomatic Go patterns, it provides a type-safe, performant interface perfect for microservices and high-performance applications.
Installation
go get github.com/ujeebu/ujeebu-goRequirements:
- Go 1.18 or higher
- Standard library
net/httppackage
Quick Start
package main
import (
"fmt"
"log"
"github.com/ujeebu/ujeebu-go"
)
func main() {
// Initialize with your API key
client, err := ujeebu.NewClient("your-api-key")
if err != nil {
log.Fatal(err)
}
// Scrape a website
result, credits, err := client.Scrape(ujeebu.ScrapeParams{
URL: "https://example.com",
JS: true,
ResponseType: "html",
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Credits used: %d\n", credits)
fmt.Println(result)
}Authentication
The SDK requires an API key for authentication. Get yours from the Ujeebu Dashboard.
Using Environment Variables (Recommended)
package main
import (
"os"
"log"
"github.com/ujeebu/ujeebu-go"
)
func main() {
client, err := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
if err != nil {
log.Fatal(err)
}
// Use client...
}package main
import (
"os"
"log"
"github.com/joho/godotenv"
"github.com/ujeebu/ujeebu-go"
)
func main() {
// Load .env file
err := godotenv.Load()
if err != nil {
log.Fatal("Error loading .env file")
}
client, err := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
if err != nil {
log.Fatal(err)
}
// Use client...
}WARNING — Security
Never hardcode API keys in your source code or commit them to version control.
Core Methods
Scrape()
Scrape web pages with various rendering and extraction options.
result, credits, err := client.Scrape(ujeebu.ScrapeParams{
URL: "https://example.com",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(result)result, credits, err := client.Scrape(ujeebu.ScrapeParams{
URL: "https://example.com",
JS: true,
JSTimeout: 5000,
WaitFor: ".dynamic-content",
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Credits: %d\n", credits)
fmt.Println(result)result, _, err := client.Scrape(ujeebu.ScrapeParams{
URL: "https://example.com",
ResponseType: "json",
ExtractRules: map[string]interface{}{
"title": "h1",
"articles": map[string]interface{}{
"selector": ".article",
"type": "list",
"data": map[string]string{
"headline": "h2",
"author": ".author",
},
},
},
})Extract()
Extract clean article content from web pages.
article, credits, err := client.Extract(ujeebu.ExtractParams{
URL: "https://example.com/article",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(article.Title)
fmt.Println(article.Author)
fmt.Println(article.Text)
fmt.Println(article.PubDate)article, _, err := client.Extract(ujeebu.ExtractParams{
URL: "https://example.com/article",
StripTags: "script,style,nav",
Images: true,
})Serp()
Get structured search engine results.
results, credits, err := client.GoogleSearch(ujeebu.SerpParams{
Search: "artificial intelligence",
Lang: "en",
ResultsCount: 20,
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Credits used: %d\n", credits)
for _, result := range results.OrganicResults {
fmt.Println(result.Title)
fmt.Println(result.Link)
}results, _, err := client.GoogleNewsSearch(ujeebu.SerpParams{
Search: "latest technology news",
Lang: "en",
ResultsCount: 10,
})
if err != nil {
log.Fatal(err)
}
for _, article := range results.News {
fmt.Println(article.Title)
}Preview()
Generate preview cards for URLs.
card, credits, err := client.Preview(ujeebu.PreviewParams{
URL: "https://example.com/article",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(card.Title)
fmt.Println(card.Description)
fmt.Println(card.Image)AIScrape()
Extract structured data using AI-powered natural language prompts.
result, credits, err := client.AIScrape(ujeebu.AIScrapeParams{
URL: "https://example.com/product",
Prompt: "Extract the product name, price, and rating",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Data)result, _, err := client.AIScrape(ujeebu.AIScrapeParams{
URL: "https://example.com/product",
Prompt: "Extract product details",
Schema: map[string]any{
"type": "object",
"properties": map[string]any{
"name": map[string]any{"type": "string"},
"price": map[string]any{"type": "number"},
},
"required": []string{"name", "price"},
},
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Name: %v, Price: %v\n", result.Data["name"], result.Data["price"])AutoExtract()
Automatically extract structured data from any web page without writing prompts or selectors.
result, credits, err := client.AutoExtract(ujeebu.AutoExtractParams{
URL: "https://example.com/article",
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Page type: %s\n", result.PageType)
fmt.Println(result.Data)result, _, err := client.AutoExtract(ujeebu.AutoExtractParams{
URL: "https://example.com/article",
HTML: existingHTML, // Skip fetching if you already have HTML
})autoProxy := false
js := false
captcha := false
result, credits, err := client.AutoExtract(ujeebu.AutoExtractParams{
URL: "https://example.com/product",
ProxyType: "premium",
AutoProxy: &autoProxy,
JS: &js,
ScrollDown: true,
WaitFor: ".product-details",
WaitForTimeout: 5000,
Timeout: 60000,
ForceRefresh: true,
AutoCaptchaSolve: &captcha,
Provider: "openai",
Model: "gpt-4o",
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Page type: %s, Credits: %d\n", result.PageType, credits)
fmt.Println(result.Data)Markdown()
Convert web pages to clean, LLM-optimized markdown.
result, credits, err := client.Markdown(ujeebu.MarkdownParams{
URL: "https://example.com/article",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Markdown)result, _, err := client.Markdown(ujeebu.MarkdownParams{
URL: "https://docs.example.com/guide",
Filter: "bm25",
Query: "installation instructions",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Markdown)
fmt.Println(result.References)citations := true
js := true
result, credits, err := client.Markdown(ujeebu.MarkdownParams{
URL: "https://example.com/spa-page",
Filter: "fit",
Citations: &citations,
JS: &js,
Wait: 3000,
WaitForSelector: ".content-loaded",
Timeout: 120,
ProxyType: "premium",
})
if err != nil {
log.Fatal(err)
}
fmt.Println(result.Markdown)
fmt.Println(result.FitMarkdown)
fmt.Println(result.MarkdownWithCitations)
fmt.Println(result.References)
fmt.Printf("Credits used: %d\n", credits)Convenience Methods
GetPDF()
Generate a PDF of a web page.
pdfBytes, _, err := client.GetPDF(ujeebu.ScrapeParams{
URL: "https://example.com",
JS: true,
WaitFor: "2000",
})
if err != nil {
log.Fatal(err)
}
err = os.WriteFile("page.pdf", pdfBytes, 0644)
if err != nil {
log.Fatal(err)
}GetScreenshot()
Capture a screenshot of a web page.
imgBytes, _, err := client.GetScreenshot(ujeebu.ScrapeParams{
URL: "https://example.com",
JS: true,
ScreenshotFullpage: true,
})
if err != nil {
log.Fatal(err)
}
err = os.WriteFile("screenshot.png", imgBytes, 0644)imgBytes, _, err := client.GetScreenshot(ujeebu.ScrapeParams{
URL: "https://example.com",
ScreenshotPartial: ".hero-section",
})
err = os.WriteFile("hero.png", imgBytes, 0644)Scrape Parameters
| Parameter | Type | Required | Default | Description |
|---|---|---|---|---|
URL |
string |
Yes | The URL to scrape. | |
JS |
bool |
No | false |
Enable JavaScript rendering. |
ResponseType |
string |
No | html |
Output format: 'html', 'screenshot', 'pdf', 'raw'. |
JSONOutput |
bool |
No | false |
When true, returns a JSON response instead of raw content. |
Timeout |
int |
No | 60 |
Maximum number of seconds before request timeout. |
WaitFor |
string |
No | "" |
CSS selector, milliseconds, or base64-encoded JS callable to wait for. |
WaitForTimeout |
int |
No | 0 |
Timeout in milliseconds for the WaitFor parameter. |
JSTimeout |
int |
No | 30000 |
Timeout for JavaScript execution in milliseconds. |
Device |
string |
No | desktop |
Device to emulate: 'desktop', 'mobile', or specific device name. |
ExtractRules |
map[string]interface{} |
No | nil |
Rules for structured data extraction using CSS selectors. |
ProxyType |
string |
No | rotating |
Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'mobile', 'custom'. |
ProxyCountry |
string |
No | US |
Country ISO code when using premium proxy. |
AutoProxy |
bool |
No | false |
Automatically try different proxies until one succeeds. |
ProxySession |
string |
No | "" |
Alphanumeric identifier to route requests through the same proxy instance. |
AutoCaptchaSolve |
bool |
No | false |
Enable automatic CAPTCHA detection and solving. |
AutoCaptchaSolveTimeout |
int |
No | 120000 |
Timeout in milliseconds for CAPTCHA solving. |
AI Scraper Parameters
| Parameter | Type | Required | Default | Description |
|---|---|---|---|---|
URL |
string |
Yes | The URL to scrape. | |
Prompt |
string |
No | Natural language instruction describing what data to extract. Required unless Schema is provided. | |
Schema |
map[string]any |
No | JSON schema defining the expected structure of extracted data. When provided, Prompt becomes optional. | |
Temperature |
*float64 |
No | 0.0 |
LLM temperature (0.0-1.0). Lower = more deterministic. |
JS |
bool |
No | true |
Enable JavaScript rendering. |
ProxyType |
string |
No | auto |
Proxy type: 'rotating', 'advanced', 'premium', 'residential'. Auto-selects if not specified. |
ProxyCountry |
string |
No | "" |
ISO country code for proxy location. |
Timeout |
int |
No | 120 |
Request timeout in seconds. |
WaitFor |
string |
No | "" |
CSS selector or milliseconds to wait before extraction. |
Markdown Parameters
| Parameter | Type | Required | Default | Description |
|---|---|---|---|---|
URL |
string |
Yes | The URL to convert to markdown. | |
Filter |
string |
No | fit |
Content filter: 'raw' (full page), 'fit' (main content), 'bm25' (relevance-ranked with query). |
Query |
string |
No | "" |
Search query for BM25 relevance filtering. Required when Filter is 'bm25'. |
Citations |
*bool |
No | nil |
Include citation references in the markdown output. |
JS |
*bool |
No | nil (defaults to true server-side) |
Enable JavaScript rendering for dynamic pages. |
Wait |
int |
No | 0 |
Milliseconds to wait after page load before conversion. |
WaitForSelector |
string |
No | "" |
CSS selector to wait for before conversion. |
Timeout |
int |
No | 60 |
Request timeout in seconds. |
ProxyType |
string |
No | "" (auto_proxy) |
Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'residential_us', 'residential_geo'. If not set, auto_proxy selects the best proxy automatically. |
Proxy |
string |
No | "" |
Custom proxy URL to use for the request. |
AutoCaptchaSolve |
*bool |
No | nil (defaults to true server-side) |
Enable automatic CAPTCHA detection and solving. |
AutoCaptchaSolveTimeout |
int |
No | 0 |
Timeout for CAPTCHA solving in milliseconds. |
CustomHeaders |
map[string]string |
No | nil |
Custom headers to forward with the request (automatically prefixed with UJB-). |
AutoExtract Parameters
| Parameter | Type | Required | Default | Description |
|---|---|---|---|---|
URL |
string |
Yes | The URL to extract data from. | |
HTML |
string |
No | "" |
Pre-fetched HTML content. Skips fetching step when provided. |
ProxyType |
string |
No | "" (auto_proxy) |
Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'residential_us', 'residential_geo'. |
AutoProxy |
*bool |
No | nil (defaults to true server-side) |
Automatic proxy selection for best results. |
JS |
*bool |
No | nil (defaults to true server-side) |
Enable JavaScript rendering. |
ScrollDown |
bool |
No | false |
Scroll down the page before extraction. |
WaitFor |
string |
No | "" |
CSS selector to wait for before extraction. |
WaitForTimeout |
int |
No | 30000 |
Timeout in milliseconds for the WaitFor selector. |
Timeout |
int |
No | 120000 |
Overall request timeout in milliseconds. |
ForceRefresh |
bool |
No | false |
Force regeneration of extraction rules even if cached. |
AutoCaptchaSolve |
*bool |
No | nil (defaults to true server-side) |
Enable automatic CAPTCHA detection and solving. |
AutoCaptchaSolveTimeout |
int |
No | 0 |
Timeout in milliseconds for CAPTCHA solving. |
Provider |
string |
No | "" |
LLM provider for rule generation (e.g., 'openai', 'anthropic', 'google'). |
Model |
string |
No | "" |
LLM model to use for rule generation. |
Error Handling
result, credits, err := client.Scrape(ujeebu.ScrapeParams{
URL: "https://example.com",
})
if err != nil {
switch e := err.(type) {
case *ujeebu.APIError:
fmt.Printf("API Error: %d - %s\n", e.StatusCode, e.Message)
case *ujeebu.NetworkError:
fmt.Printf("Network Error: %s\n", e.Error())
case *ujeebu.TimeoutError:
fmt.Println("Request timed out")
default:
fmt.Printf("Unknown error: %s\n", err.Error())
}
return
}
fmt.Println(result)Concurrent Scraping
Go's goroutines make it easy to scrape multiple URLs concurrently:
package main
import (
"fmt"
"sync"
"github.com/ujeebu/ujeebu-go"
)
func main() {
client, _ := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
urls := []string{
"https://example1.com",
"https://example2.com",
"https://example3.com",
}
var wg sync.WaitGroup
results := make(chan string, len(urls))
for _, url := range urls {
wg.Add(1)
go func(url string) {
defer wg.Done()
result, _, err := client.Scrape(ujeebu.ScrapeParams{URL: url})
if err != nil {
results <- fmt.Sprintf("Error: %s", err)
return
}
results <- fmt.Sprintf("Got %d bytes from %s", len(result), url)
}(url)
}
wg.Wait()
close(results)
for result := range results {
fmt.Println(result)
}
}Context Support
Use context for timeout and cancellation:
package main
import (
"context"
"time"
"github.com/ujeebu/ujeebu-go"
)
func main() {
client, _ := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
// Create context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, _, err := client.ScrapeWithContext(ctx, ujeebu.ScrapeParams{
URL: "https://example.com",
JS: true,
})
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
fmt.Println("Request timed out")
} else {
fmt.Printf("Error: %s\n", err)
}
return
}
fmt.Println(result)
}Spin up an API key in 60 seconds
Free tier: 5,000 credits, no card, full access to every endpoint on this page.