Start a new crawl
curl --request POST \
--url https://api.olostep.com/v1/crawls \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"start_url": "<string>",
"max_pages": 123,
"storage": {
"expires_in": "7d"
},
"include_urls": [
"<string>"
],
"exclude_urls": [
"<string>"
],
"max_depth": 123,
"include_external": true,
"include_subdomain": true,
"search_query": "<string>",
"top_n": 123,
"webhook": "<string>",
"timeout": 123,
"follow_robots_txt": true,
"scrape_options": {
"formats": [
"markdown",
"screenshot"
],
"parser": "@olostep/extract-emails"
}
}
'import requests
url = "https://api.olostep.com/v1/crawls"
payload = {
"start_url": "<string>",
"max_pages": 123,
"storage": { "expires_in": "7d" },
"include_urls": ["<string>"],
"exclude_urls": ["<string>"],
"max_depth": 123,
"include_external": True,
"include_subdomain": True,
"search_query": "<string>",
"top_n": 123,
"webhook": "<string>",
"timeout": 123,
"follow_robots_txt": True,
"scrape_options": {
"formats": ["markdown", "screenshot"],
"parser": "@olostep/extract-emails"
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
start_url: '<string>',
max_pages: 123,
storage: {expires_in: '7d'},
include_urls: ['<string>'],
exclude_urls: ['<string>'],
max_depth: 123,
include_external: true,
include_subdomain: true,
search_query: '<string>',
top_n: 123,
webhook: '<string>',
timeout: 123,
follow_robots_txt: true,
scrape_options: {formats: ['markdown', 'screenshot'], parser: '@olostep/extract-emails'}
})
};
fetch('https://api.olostep.com/v1/crawls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/crawls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'start_url' => '<string>',
'max_pages' => 123,
'storage' => [
'expires_in' => '7d'
],
'include_urls' => [
'<string>'
],
'exclude_urls' => [
'<string>'
],
'max_depth' => 123,
'include_external' => true,
'include_subdomain' => true,
'search_query' => '<string>',
'top_n' => 123,
'webhook' => '<string>',
'timeout' => 123,
'follow_robots_txt' => true,
'scrape_options' => [
'formats' => [
'markdown',
'screenshot'
],
'parser' => '@olostep/extract-emails'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/crawls"
payload := strings.NewReader("{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/crawls")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/crawls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}"
response = http.request(request)
puts response.read_body{
"storage": {
"expires_in": "7d"
},
"id": "<string>",
"object": "<string>",
"status": "<string>",
"created": 123,
"start_date": "<string>",
"start_url": "<string>",
"max_pages": 123,
"max_depth": 123,
"exclude_urls": [
"<string>"
],
"include_urls": [
"<string>"
],
"include_external": true,
"search_query": "<string>",
"top_n": 123,
"current_depth": 123,
"pages_count": 123,
"webhook": "<string>",
"follow_robots_txt": true,
"credits_consumed": 123,
"cost_usd": 123
}Crawls
Create Crawl
Starts a new crawl. You receive a id to track the progress. The operation may take 1-10 mins depending upon the site and depth and pages parameters.
POST
/
v1
/
crawls
Start a new crawl
curl --request POST \
--url https://api.olostep.com/v1/crawls \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"start_url": "<string>",
"max_pages": 123,
"storage": {
"expires_in": "7d"
},
"include_urls": [
"<string>"
],
"exclude_urls": [
"<string>"
],
"max_depth": 123,
"include_external": true,
"include_subdomain": true,
"search_query": "<string>",
"top_n": 123,
"webhook": "<string>",
"timeout": 123,
"follow_robots_txt": true,
"scrape_options": {
"formats": [
"markdown",
"screenshot"
],
"parser": "@olostep/extract-emails"
}
}
'import requests
url = "https://api.olostep.com/v1/crawls"
payload = {
"start_url": "<string>",
"max_pages": 123,
"storage": { "expires_in": "7d" },
"include_urls": ["<string>"],
"exclude_urls": ["<string>"],
"max_depth": 123,
"include_external": True,
"include_subdomain": True,
"search_query": "<string>",
"top_n": 123,
"webhook": "<string>",
"timeout": 123,
"follow_robots_txt": True,
"scrape_options": {
"formats": ["markdown", "screenshot"],
"parser": "@olostep/extract-emails"
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
start_url: '<string>',
max_pages: 123,
storage: {expires_in: '7d'},
include_urls: ['<string>'],
exclude_urls: ['<string>'],
max_depth: 123,
include_external: true,
include_subdomain: true,
search_query: '<string>',
top_n: 123,
webhook: '<string>',
timeout: 123,
follow_robots_txt: true,
scrape_options: {formats: ['markdown', 'screenshot'], parser: '@olostep/extract-emails'}
})
};
fetch('https://api.olostep.com/v1/crawls', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/crawls",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'start_url' => '<string>',
'max_pages' => 123,
'storage' => [
'expires_in' => '7d'
],
'include_urls' => [
'<string>'
],
'exclude_urls' => [
'<string>'
],
'max_depth' => 123,
'include_external' => true,
'include_subdomain' => true,
'search_query' => '<string>',
'top_n' => 123,
'webhook' => '<string>',
'timeout' => 123,
'follow_robots_txt' => true,
'scrape_options' => [
'formats' => [
'markdown',
'screenshot'
],
'parser' => '@olostep/extract-emails'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/crawls"
payload := strings.NewReader("{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/crawls")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/crawls")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"start_url\": \"<string>\",\n \"max_pages\": 123,\n \"storage\": {\n \"expires_in\": \"7d\"\n },\n \"include_urls\": [\n \"<string>\"\n ],\n \"exclude_urls\": [\n \"<string>\"\n ],\n \"max_depth\": 123,\n \"include_external\": true,\n \"include_subdomain\": true,\n \"search_query\": \"<string>\",\n \"top_n\": 123,\n \"webhook\": \"<string>\",\n \"timeout\": 123,\n \"follow_robots_txt\": true,\n \"scrape_options\": {\n \"formats\": [\n \"markdown\",\n \"screenshot\"\n ],\n \"parser\": \"@olostep/extract-emails\"\n }\n}"
response = http.request(request)
puts response.read_body{
"storage": {
"expires_in": "7d"
},
"id": "<string>",
"object": "<string>",
"status": "<string>",
"created": 123,
"start_date": "<string>",
"start_url": "<string>",
"max_pages": 123,
"max_depth": 123,
"exclude_urls": [
"<string>"
],
"include_urls": [
"<string>"
],
"include_external": true,
"search_query": "<string>",
"top_n": 123,
"current_depth": 123,
"pages_count": 123,
"webhook": "<string>",
"follow_robots_txt": true,
"credits_consumed": 123,
"cost_usd": 123
}Get notified on completion: Pass the
webhook parameter with your endpoint URL to receive an HTTP POST when the crawl completes. See Webhooks for details.Result retention: Pass
storage.expires_in to keep this crawl’s stored files longer than the default of 7 days. Values: 7d, 10d, 30d, 60d, 90d, 180d, 365d, or never. See Storage for details.Authorizations
Bearer authentication header of the form Bearer , where is your auth token.
Body
application/json
The starting point of the crawl.
Maximum number of pages to crawl. Recommended for most use cases like crawling an entire website.
Controls how long this crawl's stored files (its crawled pages) are retained. When omitted, they are kept for 7 days. The value in effect is returned in the response as storage.expires_in. See the Storage section in the feature docs. An invalid value returns 400 with code invalid_storage_expires_in.
Show child attributes
Show child attributes
URL path patterns to include in the crawl using glob syntax.
Defaults to /** which includes all URLs. Use patterns like /blog/** to crawl specific sections (e.g., only blog pages), /products/*.html for product pages, or multiple patterns for different sections. Supports standard glob features like * (any characters) and ** (recursive matching).
URL path names in glob pattern to exclude. For example: /careers/**. Excluded URLs will supersede included URLs.
Maximum depth of the crawl. Useful to extract only up to n-degree of links.
Crawl first-degree external links.
Include subdomains of the website. false by default.
An optional search query to find specific links and also sort the results by relevance.
An optional number to only crawl the top N most relevant links on every page as per search query.
End the crawl after n seconds with the pages completed until then. May take ~10s extra from provided timeout.
Whether to respect robots.txt rules. If set to false, the crawler will scrape the website regardless of robots.txt disallow directives. true by default.
Controls what each individual page scrape requests from the Olostep API. All fields are optional.
Show child attributes
Show child attributes
Response
Crawl started successfully.
Retention applied to this crawl's stored files.
Show child attributes
Show child attributes
Crawl ID
The kind of object. "crawl" for this endpoint.
in_progress or completed
Created time in epoch
Created time in date
The current depth of the crawl process.
Count of pages crawled
Number of credits consumed by this request. Populated after execution completes. Credits are the source of truth for billing.
Estimated cost in USD for this request. Populated after execution completes. Calculated from credits consumed and your plan rate — 99% accurate, but credits_consumed is the authoritative value.
Was this page helpful?