Create Batch - Olostep Docs

Start a new batch

curl --request POST \
  --url https://api.olostep.com/v1/batches \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "items": [
    {
      "custom_id": "product-123",
      "url": "https://example.com/product/123",
      "metadata": {
        "source": "catalog_sync",
        "priority": "high"
      }
    },
    {
      "custom_id": "product-456",
      "url": "https://example.com/product/456"
    }
  ],
  "country": "US",
  "metadata": {
    "batch_name": "Q1 Product Sync",
    "initiated_by": "automation"
  }
}
'

import requests

url = "https://api.olostep.com/v1/batches"

payload = {
    "items": [
        {
            "custom_id": "product-123",
            "url": "https://example.com/product/123",
            "metadata": {
                "source": "catalog_sync",
                "priority": "high"
            }
        },
        {
            "custom_id": "product-456",
            "url": "https://example.com/product/456"
        }
    ],
    "country": "US",
    "metadata": {
        "batch_name": "Q1 Product Sync",
        "initiated_by": "automation"
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    items: [
      {
        custom_id: 'product-123',
        url: 'https://example.com/product/123',
        metadata: {source: 'catalog_sync', priority: 'high'}
      },
      {custom_id: 'product-456', url: 'https://example.com/product/456'}
    ],
    country: 'US',
    metadata: {batch_name: 'Q1 Product Sync', initiated_by: 'automation'}
  })
};

fetch('https://api.olostep.com/v1/batches', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.olostep.com/v1/batches",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'items' => [
        [
                'custom_id' => 'product-123',
                'url' => 'https://example.com/product/123',
                'metadata' => [
                                'source' => 'catalog_sync',
                                'priority' => 'high'
                ]
        ],
        [
                'custom_id' => 'product-456',
                'url' => 'https://example.com/product/456'
        ]
    ],
    'country' => 'US',
    'metadata' => [
        'batch_name' => 'Q1 Product Sync',
        'initiated_by' => 'automation'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.olostep.com/v1/batches"

	payload := strings.NewReader("{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/batches")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.olostep.com/v1/batches")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "batch_abc123def456",
  "object": "batch",
  "status": "in_progress",
  "created": 1704067200,
  "total_urls": 2,
  "completed_urls": 0,
  "country": "US",
  "metadata": {
    "batch_name": "Q1 Product Sync",
    "initiated_by": "automation"
  }
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "validation_error",
  "type": "https://docs.olostep.com/api-reference/errors/bad_request",
  "status": 400,
  "title": "Bad Request",
  "detail": "The 'items' array is required and must contain at least one item.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "invalid_api_key",
  "type": "https://docs.olostep.com/api-reference/errors/unauthorized",
  "status": 401,
  "title": "Unauthorized",
  "detail": "Your API key is invalid. Try double-checking it or reaching out to info@olostep.com if you're facing issues.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "credits_exhausted",
  "type": "https://docs.olostep.com/api-reference/errors/payment_required",
  "status": 402,
  "title": "Payment Required",
  "detail": "You have consumed all available credits. Please upgrade your plan from the dashboard: https://www.olostep.com/auth/",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "access_denied",
  "type": "https://docs.olostep.com/api-reference/errors/forbidden",
  "status": 403,
  "title": "Forbidden",
  "detail": "You don't have access to this feature. Please reach out to info@olostep.com to get approved",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "idempotency_key_in_progress",
  "type": "https://docs.olostep.com/api-reference/errors/idempotency_error",
  "status": 409,
  "title": "Idempotency Error",
  "detail": "A request with this idempotency key is currently being processed. Please wait and retry.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "idempotency_key_reuse",
  "type": "https://docs.olostep.com/api-reference/errors/unprocessable_entity",
  "status": 422,
  "title": "Unprocessable Entity",
  "detail": "A request with this idempotency key was already made with different parameters. Idempotency keys must be unique per request.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "internal_server_error",
  "type": "https://docs.olostep.com/api-reference/errors/internal_error",
  "status": 500,
  "title": "Internal Server Error",
  "detail": "An unexpected error occurred",
  "created": 1704067200,
  "metadata": {}
}

POST

batches

Start a new batch

curl --request POST \
  --url https://api.olostep.com/v1/batches \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "items": [
    {
      "custom_id": "product-123",
      "url": "https://example.com/product/123",
      "metadata": {
        "source": "catalog_sync",
        "priority": "high"
      }
    },
    {
      "custom_id": "product-456",
      "url": "https://example.com/product/456"
    }
  ],
  "country": "US",
  "metadata": {
    "batch_name": "Q1 Product Sync",
    "initiated_by": "automation"
  }
}
'

import requests

url = "https://api.olostep.com/v1/batches"

payload = {
    "items": [
        {
            "custom_id": "product-123",
            "url": "https://example.com/product/123",
            "metadata": {
                "source": "catalog_sync",
                "priority": "high"
            }
        },
        {
            "custom_id": "product-456",
            "url": "https://example.com/product/456"
        }
    ],
    "country": "US",
    "metadata": {
        "batch_name": "Q1 Product Sync",
        "initiated_by": "automation"
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    items: [
      {
        custom_id: 'product-123',
        url: 'https://example.com/product/123',
        metadata: {source: 'catalog_sync', priority: 'high'}
      },
      {custom_id: 'product-456', url: 'https://example.com/product/456'}
    ],
    country: 'US',
    metadata: {batch_name: 'Q1 Product Sync', initiated_by: 'automation'}
  })
};

fetch('https://api.olostep.com/v1/batches', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.olostep.com/v1/batches",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'items' => [
        [
                'custom_id' => 'product-123',
                'url' => 'https://example.com/product/123',
                'metadata' => [
                                'source' => 'catalog_sync',
                                'priority' => 'high'
                ]
        ],
        [
                'custom_id' => 'product-456',
                'url' => 'https://example.com/product/456'
        ]
    ],
    'country' => 'US',
    'metadata' => [
        'batch_name' => 'Q1 Product Sync',
        'initiated_by' => 'automation'
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.olostep.com/v1/batches"

	payload := strings.NewReader("{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/batches")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.olostep.com/v1/batches")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"items\": [\n    {\n      \"custom_id\": \"product-123\",\n      \"url\": \"https://example.com/product/123\",\n      \"metadata\": {\n        \"source\": \"catalog_sync\",\n        \"priority\": \"high\"\n      }\n    },\n    {\n      \"custom_id\": \"product-456\",\n      \"url\": \"https://example.com/product/456\"\n    }\n  ],\n  \"country\": \"US\",\n  \"metadata\": {\n    \"batch_name\": \"Q1 Product Sync\",\n    \"initiated_by\": \"automation\"\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "id": "batch_abc123def456",
  "object": "batch",
  "status": "in_progress",
  "created": 1704067200,
  "total_urls": 2,
  "completed_urls": 0,
  "country": "US",
  "metadata": {
    "batch_name": "Q1 Product Sync",
    "initiated_by": "automation"
  }
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "validation_error",
  "type": "https://docs.olostep.com/api-reference/errors/bad_request",
  "status": 400,
  "title": "Bad Request",
  "detail": "The 'items' array is required and must contain at least one item.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "invalid_api_key",
  "type": "https://docs.olostep.com/api-reference/errors/unauthorized",
  "status": 401,
  "title": "Unauthorized",
  "detail": "Your API key is invalid. Try double-checking it or reaching out to info@olostep.com if you're facing issues.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "credits_exhausted",
  "type": "https://docs.olostep.com/api-reference/errors/payment_required",
  "status": 402,
  "title": "Payment Required",
  "detail": "You have consumed all available credits. Please upgrade your plan from the dashboard: https://www.olostep.com/auth/",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "access_denied",
  "type": "https://docs.olostep.com/api-reference/errors/forbidden",
  "status": 403,
  "title": "Forbidden",
  "detail": "You don't have access to this feature. Please reach out to info@olostep.com to get approved",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "idempotency_key_in_progress",
  "type": "https://docs.olostep.com/api-reference/errors/idempotency_error",
  "status": 409,
  "title": "Idempotency Error",
  "detail": "A request with this idempotency key is currently being processed. Please wait and retry.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "idempotency_key_reuse",
  "type": "https://docs.olostep.com/api-reference/errors/unprocessable_entity",
  "status": 422,
  "title": "Unprocessable Entity",
  "detail": "A request with this idempotency key was already made with different parameters. Idempotency keys must be unique per request.",
  "created": 1704067200,
  "metadata": {}
}

{
  "id": "error_abc123",
  "object": "error",
  "code": "internal_server_error",
  "type": "https://docs.olostep.com/api-reference/errors/internal_error",
  "status": 500,
  "title": "Internal Server Error",
  "detail": "An unexpected error occurred",
  "created": 1704067200,
  "metadata": {}
}

Get notified on completion: Pass the webhook parameter with your endpoint URL to receive an HTTP POST when the batch completes. See Webhooks for details.

Attach custom data: Use the metadata parameter to store key-value pairs. Supported at two levels:

Batch-level — on the request body
Item-level — on each item in the items array

See Metadata for details.

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer , where is your auth token.

Body

application/json

items

object[]

required

Array of items to be processed in the batch.

Show child attributes

country

string

Country for the batch execution. Provide in ISO 3166-1 alpha-2 codes like US(USA), IN(India), etc

parser

object

You can use this parameter to specify the parser to use. Parsers are useful to extract structured content from web pages. Olostep has a few parsers built in for most common web pages, and you can also create your own parsers.

Show child attributes

links_on_page

object

Get all the links present on each page in the batch. Links are always returned as absolute URLs.

Show child attributes

metadata

object

Set of key-value pairs for storing additional information about an object. Follows Stripe's approach with validation rules: max 50 keys, key max 40 characters (no square brackets), value max 500 characters, all values stored as strings.

Show child attributes

Example:

{
  "order_id": "12345",
  "customer_name": "John Doe",
  "priority": "high",
  "processed": "true"
}

webhook

string<uri>

HTTPS URL to receive a POST request when the batch completes. Must be a publicly accessible URL using http:// or https:// protocol. Cannot point to localhost or private IP addresses. See Webhooks for payload format and retry behavior.

Response

Batch started successfully.

string

Batch ID

object

string

The kind of object. "batch" for this endpoint.

status

string

in_progress or completed

created

number

Created epoch

total_urls

number

Count of URLs in the batch

completed_urls

number

Count of completed URLs

parser

string

country

string

metadata

object

Show child attributes

Example:

{
  "order_id": "12345",
  "customer_name": "John Doe",
  "priority": "high",
  "processed": "true"
}

webhook

string

Webhook URL to receive completion notification

credits_consumed

integer | null

Number of credits consumed by this request. Populated after execution completes. Credits are the source of truth for billing.

cost_usd

number | null

Estimated cost in USD for this request. Populated after execution completes. Calculated from credits consumed and your plan rate — 99% accurate, but credits_consumed is the authoritative value.

Get Scrape Batch Info