curl --request GET \
--url https://api.olostep.com/v1/crawls/{crawl_id}/pages \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.olostep.com/v1/crawls/{crawl_id}/pages"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.olostep.com/v1/crawls/{crawl_id}/pages', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/crawls/{crawl_id}/pages",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/crawls/{crawl_id}/pages"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.olostep.com/v1/crawls/{crawl_id}/pages")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/crawls/{crawl_id}/pages")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"crawl_id": "<string>",
"object": "<string>",
"status": "<string>",
"search_query": "<string>",
"pages_count": 123,
"pages": [
{
"id": "<string>",
"retrieve_id": "<string>",
"url": "<string>",
"is_external": true,
"html_content": "<string>",
"markdown_content": "<string>"
}
],
"metadata": {
"external_urls": [
"<string>"
],
"failed_urls": [
"<string>"
]
},
"cursor": 123
}Crawl Pages
Fetches the list of pages for a specific crawl.
curl --request GET \
--url https://api.olostep.com/v1/crawls/{crawl_id}/pages \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.olostep.com/v1/crawls/{crawl_id}/pages"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.olostep.com/v1/crawls/{crawl_id}/pages', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/crawls/{crawl_id}/pages",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/crawls/{crawl_id}/pages"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.olostep.com/v1/crawls/{crawl_id}/pages")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/crawls/{crawl_id}/pages")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"crawl_id": "<string>",
"object": "<string>",
"status": "<string>",
"search_query": "<string>",
"pages_count": 123,
"pages": [
{
"id": "<string>",
"retrieve_id": "<string>",
"url": "<string>",
"is_external": true,
"html_content": "<string>",
"markdown_content": "<string>"
}
],
"metadata": {
"external_urls": [
"<string>"
],
"failed_urls": [
"<string>"
]
},
"cursor": 123
}Authorizations
Bearer authentication header of the form Bearer , where is your auth token.
Path Parameters
The ID of the crawl to retrieve the list of URLs for.
Query Parameters
Optional integer representing the index to start fetching content from. Useful to paginate until all URLs are fetched. Start with 0, then provide response['cursor'] value of the last request.
Optional integer to limit the number of results returned. Recommended 10-50 results at a time. Paginated using cursor. Maximum 10MB of content can be fetched in a single request.
An optional search query to sort the results by relevance. Uses the original search_query by default if provided.
Deprecated: Use /retrieve endpoint with retrieve_id.
Array of formats to fetch (e.g., ["html", "markdown"]).
html, markdown Response
Successful response with the list of URLs.
Crawl ID
The kind of object. "crawl" for this endpoint.
in_progress or completed
Show child attributes
Show child attributes
Show child attributes
Show child attributes
To be passed in the query in next request to get the next items.
Was this page helpful?