API Documentation

# pip install requests
# https://pypi.org/project/requests/
import requests

proxy_url = "http://YOUR_API_KEY:js=true&proxy=residential@proxy.webscraping.ai:8888"

response = requests.get(
    "https://example.com",
    proxies={"http": proxy_url, "https": proxy_url},
    verify=False,  # proxy uses a self-signed cert
)
print(response.text)

# pip install scrapy
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpproxy
import scrapy

PROXY = "http://YOUR_API_KEY:js=true&proxy=residential@proxy.webscraping.ai:8888"

class ExampleSpider(scrapy.Spider):
    name = "example"
    custom_settings = {
        # the proxy presents a self-signed cert
        "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": False,
    }

    def start_requests(self):
        yield scrapy.Request(
            "https://example.com",
            meta={"proxy": PROXY},
            callback=self.parse,
        )

    def parse(self, response):
        yield {"title": response.css("title::text").get()}

# pip install selenium-wire
# https://pypi.org/project/selenium-wire/  (vanilla Selenium can't pass proxy credentials)
from seleniumwire import webdriver

PROXY = "http://YOUR_API_KEY:js=true&proxy=residential@proxy.webscraping.ai:8888"

seleniumwire_options = {
    "proxy": {"http": PROXY, "https": PROXY, "no_proxy": "localhost,127.0.0.1"},
    "verify_ssl": False,
}

driver = webdriver.Chrome(seleniumwire_options=seleniumwire_options)
driver.get("https://example.com")
print(driver.page_source)
driver.quit()

// npm install playwright
// https://playwright.dev/docs/network#http-proxy
const { chromium } = require('playwright');

(async () => {
  const browser = await chromium.launch({
    proxy: {
      server: 'http://proxy.webscraping.ai:8888',
      username: 'YOUR_API_KEY',
      password: 'js=true&proxy=residential',
    },
  });
  const context = await browser.newContext({ ignoreHTTPSErrors: true });
  const page = await context.newPage();
  await page.goto('https://example.com');
  console.log(await page.content());
  await browser.close();
})();

// npm install puppeteer
// https://pptr.dev/api/puppeteer.page.authenticate
const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({
    args: [
      '--proxy-server=proxy.webscraping.ai:8888',
      '--ignore-certificate-errors',
    ],
  });
  const page = await browser.newPage();
  await page.authenticate({
    username: 'YOUR_API_KEY',
    password: 'js=true&proxy=residential',
  });
  await page.goto('https://example.com');
  console.log(await page.content());
  await browser.close();
})();

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
answer = client.question(
    "https://news.ycombinator.com",
    question="What are the top 3 stories on this page?",
)
print(answer)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const answer = await client.question({
  url: 'https://news.ycombinator.com',
  question: 'What are the top 3 stories on this page?',
});
console.log(answer);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$answer = $client->question(
    'https://news.ycombinator.com',
    'What are the top 3 stories on this page?',
);
echo $answer;

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
answer = client.question(
  'https://news.ycombinator.com',
  question: 'What are the top 3 stories on this page?'
)
puts answer

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    answer, _ := client.Question(context.Background(), &webscrapingai.QuestionOptions{
        URL:      "https://news.ycombinator.com",
        Question: "What are the top 3 stories on this page?",
    })
    fmt.Println(answer)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.option.QuestionOptions;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
String answer = client.question(QuestionOptions.builder()
    .url("https://news.ycombinator.com")
    .question("What are the top 3 stories on this page?")
    .build());
System.out.println(answer);

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var answer = await client.QuestionAsync(new QuestionRequest {
    Url = "https://news.ycombinator.com",
    Question = "What are the top 3 stories on this page?",
});
Console.WriteLine(answer);

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
result = client.fields(
    "https://amazon.com/dp/B08N5WRWNW",
    fields={
        "title": "Product title",
        "price": "Current price with currency",
        "rating": "Average star rating",
    },
)
print(result)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const result = await client.fields({
  url: 'https://amazon.com/dp/B08N5WRWNW',
  fields: {
    title: 'Product title',
    price: 'Current price with currency',
    rating: 'Average star rating',
  },
});
console.log(result);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$result = $client->fields('https://amazon.com/dp/B08N5WRWNW', [
    'title'  => 'Product title',
    'price'  => 'Current price with currency',
    'rating' => 'Average star rating',
]);
print_r($result);

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
result = client.fields(
  'https://amazon.com/dp/B08N5WRWNW',
  fields: {
    title:  'Product title',
    price:  'Current price with currency',
    rating: 'Average star rating'
  }
)
puts result.inspect

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    result, _ := client.Fields(context.Background(), &webscrapingai.FieldsOptions{
        URL: "https://amazon.com/dp/B08N5WRWNW",
        Fields: map[string]string{
            "title":  "Product title",
            "price":  "Current price with currency",
            "rating": "Average star rating",
        },
    })
    fmt.Println(result.Result)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.option.FieldsOptions;
import ai.webscraping.result.FieldsResult;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
FieldsResult result = client.fields(FieldsOptions.builder()
    .url("https://amazon.com/dp/B08N5WRWNW")
    .addField("title", "Product title")
    .addField("price", "Current price with currency")
    .addField("rating", "Average star rating")
    .build());
System.out.println(result.getResult());

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var result = await client.FieldsAsync(new FieldsRequest {
    Url = "https://amazon.com/dp/B08N5WRWNW",
    Fields = new Dictionary<string, string> {
        ["title"]  = "Product title",
        ["price"]  = "Current price with currency",
        ["rating"] = "Average star rating",
    },
});
Console.WriteLine(result.Result);

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
html = client.html("https://example.com", js=True)
print(html)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const html = await client.html({ url: 'https://example.com', js: true });
console.log(html);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$html = $client->html('https://example.com', js: true);
echo $html;

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
html = client.html('https://example.com', js: true)
puts html

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    js := true
    html, _ := client.HTML(context.Background(), &webscrapingai.HTMLOptions{
        URL: "https://example.com",
        CommonOptions: webscrapingai.CommonOptions{JS: &js},
    })
    fmt.Println(html)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.option.HtmlOptions;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
String html = client.html(HtmlOptions.builder()
    .url("https://example.com")
    .js(true)
    .build());
System.out.println(html);

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var html = await client.HtmlAsync(new HtmlRequest {
    Url = "https://example.com",
    Js = true,
});
Console.WriteLine(html);

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
result = client.text("https://example.com", text_format="json")
print(result)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const result = await client.text({
  url: 'https://example.com',
  text_format: 'json',
});
console.log(result);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$result = $client->text('https://example.com', textFormat: 'json');
print_r($result);

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
result = client.text('https://example.com', text_format: 'json')
puts result.inspect

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    text, _ := client.Text(context.Background(), &webscrapingai.TextOptions{
        URL:        "https://example.com",
        TextFormat: "json",
    })
    fmt.Println(text)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.option.TextOptions;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
String text = client.text(TextOptions.builder()
    .url("https://example.com")
    .textFormat("json")
    .build());
System.out.println(text);

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var text = await client.TextAsync(new TextRequest {
    Url = "https://example.com",
    TextFormat = "json",
});
Console.WriteLine(text);

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
html = client.selected("https://example.com", selector="h1")
print(html)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const html = await client.selected({
  url: 'https://example.com',
  selector: 'h1',
});
console.log(html);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$html = $client->selected('https://example.com', 'h1');
echo $html;

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
html = client.selected('https://example.com', selector: 'h1')
puts html

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    html, _ := client.Selected(context.Background(), &webscrapingai.SelectedOptions{
        URL:      "https://example.com",
        Selector: "h1",
    })
    fmt.Println(html)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.option.SelectedOptions;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
String html = client.selected(SelectedOptions.builder()
    .url("https://example.com")
    .selector("h1")
    .build());
System.out.println(html);

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var html = await client.SelectedAsync(new SelectedRequest {
    Url = "https://example.com",
    Selector = "h1",
});
Console.WriteLine(html);

# pip install webscraping_ai
# https://pypi.org/project/webscraping-ai/
from webscraping_ai import Client

client = Client(api_key="YOUR_API_KEY")
info = client.account()
print(info)

// npm install webscraping-ai
// https://www.npmjs.com/package/webscraping-ai
import { WebScrapingAI } from 'webscraping-ai';

const client = new WebScrapingAI({ apiKey: 'YOUR_API_KEY' });
const info = await client.account();
console.log(info);

<?php
// composer require webscraping-ai/webscraping-ai-php
// https://packagist.org/packages/webscraping-ai/webscraping-ai-php
require 'vendor/autoload.php';

use WebScrapingAI\Client;

$client = new Client('YOUR_API_KEY');
$info = $client->account();
print_r($info);

# gem install webscraping_ai
# https://rubygems.org/gems/webscraping_ai
require 'webscraping_ai'

client = WebScrapingAI::Client.new(api_key: 'YOUR_API_KEY')
info = client.account
puts info.inspect

// go get github.com/webscraping-ai/webscraping-ai-go/v4
// https://pkg.go.dev/github.com/webscraping-ai/webscraping-ai-go/v4
package main

import (
    "context"
    "fmt"

    webscrapingai "github.com/webscraping-ai/webscraping-ai-go/v4"
)

func main() {
    client, _ := webscrapingai.NewClient(&webscrapingai.Config{APIKey: "YOUR_API_KEY"})
    info, _ := client.Account(context.Background())
    fmt.Printf("%s - %d calls left\n", info.Email, info.RemainingAPICalls)
}

// Maven: ai.webscraping:webscraping-ai:4.0.0
// https://central.sonatype.com/artifact/ai.webscraping/webscraping-ai
import ai.webscraping.Client;
import ai.webscraping.Config;
import ai.webscraping.result.AccountInfo;

Client client = new Client(Config.builder().apiKey("YOUR_API_KEY").build());
AccountInfo info = client.account();
System.out.println(info);

// dotnet add package WebScrapingAI
// https://www.nuget.org/packages/WebScrapingAI
using WebScrapingAI;

var client = new WebScrapingAIClient(new WebScrapingAIClientOptions { ApiKey = "YOUR_API_KEY" });
var info = await client.AccountAsync();
Console.WriteLine($"{info.Email} - {info.RemainingApiCalls} calls left");