Playground Sign in Start free
Ujeebu SDKs

Go SDK

The official Ujeebu SDK for Go. Built with idiomatic Go patterns, it provides a type-safe, performant interface perfect for microservices and high-performance applications.

Installation

go get github.com/ujeebu/ujeebu-go

Requirements:

  • Go 1.18 or higher
  • Standard library net/http package

Quick Start

package main

import (
    "fmt"
    "log"
    "github.com/ujeebu/ujeebu-go"
)

func main() {
    // Initialize with your API key
    client, err := ujeebu.NewClient("your-api-key")
    if err != nil {
        log.Fatal(err)
    }

    // Scrape a website
    result, credits, err := client.Scrape(ujeebu.ScrapeParams{
        URL: "https://example.com",
        JS:  true,
        ResponseType: "html",
    })

    if err != nil {
        log.Fatal(err)
    }

    fmt.Printf("Credits used: %d\n", credits)
    fmt.Println(result)
}

Authentication

The SDK requires an API key for authentication. Get yours from the Ujeebu Dashboard.

package main

import (
    "os"
    "log"
    "github.com/ujeebu/ujeebu-go"
)

func main() {
    client, err := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
    if err != nil {
        log.Fatal(err)
    }
    // Use client...
}
package main

import (
    "os"
    "log"
    "github.com/joho/godotenv"
    "github.com/ujeebu/ujeebu-go"
)

func main() {
    // Load .env file
    err := godotenv.Load()
    if err != nil {
        log.Fatal("Error loading .env file")
    }

    client, err := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
    if err != nil {
        log.Fatal(err)
    }
    // Use client...
}

WARNING — Security

Never hardcode API keys in your source code or commit them to version control.

Core Methods

Scrape()

Scrape web pages with various rendering and extraction options.

result, credits, err := client.Scrape(ujeebu.ScrapeParams{
    URL: "https://example.com",
})
if err != nil {
    log.Fatal(err)
}
fmt.Println(result)
result, credits, err := client.Scrape(ujeebu.ScrapeParams{
    URL:          "https://example.com",
    JS:           true,
    JSTimeout:    5000,
    WaitFor:      ".dynamic-content",
})

if err != nil {
    log.Fatal(err)
}
fmt.Printf("Credits: %d\n", credits)
fmt.Println(result)
result, _, err := client.Scrape(ujeebu.ScrapeParams{
    URL:          "https://example.com",
    ResponseType: "json",
    ExtractRules: map[string]interface{}{
        "title": "h1",
        "articles": map[string]interface{}{
            "selector": ".article",
            "type":     "list",
            "data": map[string]string{
                "headline": "h2",
                "author":   ".author",
            },
        },
    },
})

Extract()

Extract clean article content from web pages.

article, credits, err := client.Extract(ujeebu.ExtractParams{
    URL: "https://example.com/article",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(article.Title)
fmt.Println(article.Author)
fmt.Println(article.Text)
fmt.Println(article.PubDate)
article, _, err := client.Extract(ujeebu.ExtractParams{
    URL:       "https://example.com/article",
    StripTags: "script,style,nav",
    Images:    true,
})

Serp()

Get structured search engine results.

results, credits, err := client.GoogleSearch(ujeebu.SerpParams{
    Search:       "artificial intelligence",
    Lang:         "en",
    ResultsCount: 20,
})

if err != nil {
    log.Fatal(err)
}

fmt.Printf("Credits used: %d\n", credits)
for _, result := range results.OrganicResults {
    fmt.Println(result.Title)
    fmt.Println(result.Link)
}
results, _, err := client.GoogleNewsSearch(ujeebu.SerpParams{
    Search:       "latest technology news",
    Lang:         "en",
    ResultsCount: 10,
})

if err != nil {
    log.Fatal(err)
}

for _, article := range results.News {
    fmt.Println(article.Title)
}

Preview()

Generate preview cards for URLs.

card, credits, err := client.Preview(ujeebu.PreviewParams{
    URL: "https://example.com/article",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(card.Title)
fmt.Println(card.Description)
fmt.Println(card.Image)

AIScrape()

Extract structured data using AI-powered natural language prompts.

result, credits, err := client.AIScrape(ujeebu.AIScrapeParams{
    URL:    "https://example.com/product",
    Prompt: "Extract the product name, price, and rating",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(result.Data)
result, _, err := client.AIScrape(ujeebu.AIScrapeParams{
    URL:    "https://example.com/product",
    Prompt: "Extract product details",
    Schema: map[string]any{
        "type": "object",
        "properties": map[string]any{
            "name":  map[string]any{"type": "string"},
            "price": map[string]any{"type": "number"},
        },
        "required": []string{"name", "price"},
    },
})

if err != nil {
    log.Fatal(err)
}

fmt.Printf("Name: %v, Price: %v\n", result.Data["name"], result.Data["price"])

AutoExtract()

Automatically extract structured data from any web page without writing prompts or selectors.

result, credits, err := client.AutoExtract(ujeebu.AutoExtractParams{
    URL: "https://example.com/article",
})

if err != nil {
    log.Fatal(err)
}

fmt.Printf("Page type: %s\n", result.PageType)
fmt.Println(result.Data)
result, _, err := client.AutoExtract(ujeebu.AutoExtractParams{
    URL:  "https://example.com/article",
    HTML: existingHTML, // Skip fetching if you already have HTML
})
autoProxy := false
js := false
captcha := false

result, credits, err := client.AutoExtract(ujeebu.AutoExtractParams{
    URL:              "https://example.com/product",
    ProxyType:        "premium",
    AutoProxy:        &autoProxy,
    JS:               &js,
    ScrollDown:       true,
    WaitFor:          ".product-details",
    WaitForTimeout:   5000,
    Timeout:          60000,
    ForceRefresh:     true,
    AutoCaptchaSolve: &captcha,
    Provider:         "openai",
    Model:            "gpt-4o",
})

if err != nil {
    log.Fatal(err)
}

fmt.Printf("Page type: %s, Credits: %d\n", result.PageType, credits)
fmt.Println(result.Data)

Markdown()

Convert web pages to clean, LLM-optimized markdown.

result, credits, err := client.Markdown(ujeebu.MarkdownParams{
    URL: "https://example.com/article",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(result.Markdown)
result, _, err := client.Markdown(ujeebu.MarkdownParams{
    URL:    "https://docs.example.com/guide",
    Filter: "bm25",
    Query:  "installation instructions",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(result.Markdown)
fmt.Println(result.References)
citations := true
js := true

result, credits, err := client.Markdown(ujeebu.MarkdownParams{
    URL:             "https://example.com/spa-page",
    Filter:          "fit",
    Citations:       &citations,
    JS:              &js,
    Wait:            3000,
    WaitForSelector: ".content-loaded",
    Timeout:         120,
    ProxyType:       "premium",
})

if err != nil {
    log.Fatal(err)
}

fmt.Println(result.Markdown)
fmt.Println(result.FitMarkdown)
fmt.Println(result.MarkdownWithCitations)
fmt.Println(result.References)
fmt.Printf("Credits used: %d\n", credits)

Convenience Methods

GetPDF()

Generate a PDF of a web page.

pdfBytes, _, err := client.GetPDF(ujeebu.ScrapeParams{
    URL:     "https://example.com",
    JS:      true,
    WaitFor: "2000",
})

if err != nil {
    log.Fatal(err)
}

err = os.WriteFile("page.pdf", pdfBytes, 0644)
if err != nil {
    log.Fatal(err)
}

GetScreenshot()

Capture a screenshot of a web page.

imgBytes, _, err := client.GetScreenshot(ujeebu.ScrapeParams{
    URL:                "https://example.com",
    JS:                 true,
    ScreenshotFullpage: true,
})

if err != nil {
    log.Fatal(err)
}

err = os.WriteFile("screenshot.png", imgBytes, 0644)
imgBytes, _, err := client.GetScreenshot(ujeebu.ScrapeParams{
    URL:               "https://example.com",
    ScreenshotPartial: ".hero-section",
})

err = os.WriteFile("hero.png", imgBytes, 0644)

Scrape Parameters

Parameter Type Required Default Description
URL string Yes The URL to scrape.
JS bool No false Enable JavaScript rendering.
ResponseType string No html Output format: 'html', 'screenshot', 'pdf', 'raw'.
JSONOutput bool No false When true, returns a JSON response instead of raw content.
Timeout int No 60 Maximum number of seconds before request timeout.
WaitFor string No "" CSS selector, milliseconds, or base64-encoded JS callable to wait for.
WaitForTimeout int No 0 Timeout in milliseconds for the WaitFor parameter.
JSTimeout int No 30000 Timeout for JavaScript execution in milliseconds.
Device string No desktop Device to emulate: 'desktop', 'mobile', or specific device name.
ExtractRules map[string]interface{} No nil Rules for structured data extraction using CSS selectors.
ProxyType string No rotating Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'mobile', 'custom'.
ProxyCountry string No US Country ISO code when using premium proxy.
AutoProxy bool No false Automatically try different proxies until one succeeds.
ProxySession string No "" Alphanumeric identifier to route requests through the same proxy instance.
AutoCaptchaSolve bool No false Enable automatic CAPTCHA detection and solving.
AutoCaptchaSolveTimeout int No 120000 Timeout in milliseconds for CAPTCHA solving.

AI Scraper Parameters

Parameter Type Required Default Description
URL string Yes The URL to scrape.
Prompt string No Natural language instruction describing what data to extract. Required unless Schema is provided.
Schema map[string]any No JSON schema defining the expected structure of extracted data. When provided, Prompt becomes optional.
Temperature *float64 No 0.0 LLM temperature (0.0-1.0). Lower = more deterministic.
JS bool No true Enable JavaScript rendering.
ProxyType string No auto Proxy type: 'rotating', 'advanced', 'premium', 'residential'. Auto-selects if not specified.
ProxyCountry string No "" ISO country code for proxy location.
Timeout int No 120 Request timeout in seconds.
WaitFor string No "" CSS selector or milliseconds to wait before extraction.

Markdown Parameters

Parameter Type Required Default Description
URL string Yes The URL to convert to markdown.
Filter string No fit Content filter: 'raw' (full page), 'fit' (main content), 'bm25' (relevance-ranked with query).
Query string No "" Search query for BM25 relevance filtering. Required when Filter is 'bm25'.
Citations *bool No nil Include citation references in the markdown output.
JS *bool No nil (defaults to true server-side) Enable JavaScript rendering for dynamic pages.
Wait int No 0 Milliseconds to wait after page load before conversion.
WaitForSelector string No "" CSS selector to wait for before conversion.
Timeout int No 60 Request timeout in seconds.
ProxyType string No "" (auto_proxy) Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'residential_us', 'residential_geo'. If not set, auto_proxy selects the best proxy automatically.
Proxy string No "" Custom proxy URL to use for the request.
AutoCaptchaSolve *bool No nil (defaults to true server-side) Enable automatic CAPTCHA detection and solving.
AutoCaptchaSolveTimeout int No 0 Timeout for CAPTCHA solving in milliseconds.
CustomHeaders map[string]string No nil Custom headers to forward with the request (automatically prefixed with UJB-).

AutoExtract Parameters

Parameter Type Required Default Description
URL string Yes The URL to extract data from.
HTML string No "" Pre-fetched HTML content. Skips fetching step when provided.
ProxyType string No "" (auto_proxy) Proxy type: 'rotating', 'advanced', 'premium', 'residential', 'residential_us', 'residential_geo'.
AutoProxy *bool No nil (defaults to true server-side) Automatic proxy selection for best results.
JS *bool No nil (defaults to true server-side) Enable JavaScript rendering.
ScrollDown bool No false Scroll down the page before extraction.
WaitFor string No "" CSS selector to wait for before extraction.
WaitForTimeout int No 30000 Timeout in milliseconds for the WaitFor selector.
Timeout int No 120000 Overall request timeout in milliseconds.
ForceRefresh bool No false Force regeneration of extraction rules even if cached.
AutoCaptchaSolve *bool No nil (defaults to true server-side) Enable automatic CAPTCHA detection and solving.
AutoCaptchaSolveTimeout int No 0 Timeout in milliseconds for CAPTCHA solving.
Provider string No "" LLM provider for rule generation (e.g., 'openai', 'anthropic', 'google').
Model string No "" LLM model to use for rule generation.

Error Handling

result, credits, err := client.Scrape(ujeebu.ScrapeParams{
    URL: "https://example.com",
})

if err != nil {
    switch e := err.(type) {
    case *ujeebu.APIError:
        fmt.Printf("API Error: %d - %s\n", e.StatusCode, e.Message)
    case *ujeebu.NetworkError:
        fmt.Printf("Network Error: %s\n", e.Error())
    case *ujeebu.TimeoutError:
        fmt.Println("Request timed out")
    default:
        fmt.Printf("Unknown error: %s\n", err.Error())
    }
    return
}

fmt.Println(result)

Concurrent Scraping

Go's goroutines make it easy to scrape multiple URLs concurrently:

package main

import (
    "fmt"
    "sync"
    "github.com/ujeebu/ujeebu-go"
)

func main() {
    client, _ := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
    
    urls := []string{
        "https://example1.com",
        "https://example2.com",
        "https://example3.com",
    }
    
    var wg sync.WaitGroup
    results := make(chan string, len(urls))
    
    for _, url := range urls {
        wg.Add(1)
        go func(url string) {
            defer wg.Done()
            result, _, err := client.Scrape(ujeebu.ScrapeParams{URL: url})
            if err != nil {
                results <- fmt.Sprintf("Error: %s", err)
                return
            }
            results <- fmt.Sprintf("Got %d bytes from %s", len(result), url)
        }(url)
    }
    
    wg.Wait()
    close(results)
    
    for result := range results {
        fmt.Println(result)
    }
}

Context Support

Use context for timeout and cancellation:

package main

import (
    "context"
    "time"
    "github.com/ujeebu/ujeebu-go"
)

func main() {
    client, _ := ujeebu.NewClient(os.Getenv("UJEEBU_API_KEY"))
    
    // Create context with timeout
    ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
    defer cancel()
    
    result, _, err := client.ScrapeWithContext(ctx, ujeebu.ScrapeParams{
        URL: "https://example.com",
        JS:  true,
    })
    
    if err != nil {
        if ctx.Err() == context.DeadlineExceeded {
            fmt.Println("Request timed out")
        } else {
            fmt.Printf("Error: %s\n", err)
        }
        return
    }
    
    fmt.Println(result)
}
Ready to build?

Spin up an API key in 60 seconds

Free tier: 5,000 credits, no card, full access to every endpoint on this page.