Eine Webseitenscrape initiieren
curl --request POST \
--url https://api.olostep.com/v1/scrapes \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url_to_scrape": "<string>",
"wait_before_scraping": 123,
"formats": [],
"actions": [
{
"type": "wait",
"milliseconds": 1
}
],
"country": "<string>",
"remove_images": false,
"remove_class_names": [
"<string>"
],
"llm_extract": {
"schema": {}
},
"links_on_page": {
"query_to_order_links_by": "<string>",
"include_links": [
"<string>"
],
"exclude_links": [
"<string>"
]
},
"screen_size": {
"screen_width": 123,
"screen_height": 123
},
"screenshot": {
"full_page": true
},
"metadata": {},
"max_age": 0
}
'import requests
url = "https://api.olostep.com/v1/scrapes"
payload = {
"url_to_scrape": "<string>",
"wait_before_scraping": 123,
"formats": [],
"actions": [
{
"type": "wait",
"milliseconds": 1
}
],
"country": "<string>",
"remove_images": False,
"remove_class_names": ["<string>"],
"llm_extract": { "schema": {} },
"links_on_page": {
"query_to_order_links_by": "<string>",
"include_links": ["<string>"],
"exclude_links": ["<string>"]
},
"screen_size": {
"screen_width": 123,
"screen_height": 123
},
"screenshot": { "full_page": True },
"metadata": {},
"max_age": 0
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url_to_scrape: '<string>',
wait_before_scraping: 123,
formats: [],
actions: [{type: 'wait', milliseconds: 1}],
country: '<string>',
remove_images: false,
remove_class_names: ['<string>'],
llm_extract: {schema: {}},
links_on_page: {
query_to_order_links_by: '<string>',
include_links: ['<string>'],
exclude_links: ['<string>']
},
screen_size: {screen_width: 123, screen_height: 123},
screenshot: {full_page: true},
metadata: {},
max_age: 0
})
};
fetch('https://api.olostep.com/v1/scrapes', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/scrapes",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url_to_scrape' => '<string>',
'wait_before_scraping' => 123,
'formats' => [
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 1
]
],
'country' => '<string>',
'remove_images' => false,
'remove_class_names' => [
'<string>'
],
'llm_extract' => [
'schema' => [
]
],
'links_on_page' => [
'query_to_order_links_by' => '<string>',
'include_links' => [
'<string>'
],
'exclude_links' => [
'<string>'
]
],
'screen_size' => [
'screen_width' => 123,
'screen_height' => 123
],
'screenshot' => [
'full_page' => true
],
'metadata' => [
],
'max_age' => 0
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/scrapes"
payload := strings.NewReader("{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/scrapes")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/scrapes")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"object": "<string>",
"created": 123,
"metadata": {},
"url_to_scrape": "<string>",
"result": {
"html_content": "<string>",
"markdown_content": "<string>",
"text_content": "<string>",
"json_content": "<string>",
"screenshot_hosted_url": "<string>",
"html_hosted_url": "<string>",
"markdown_hosted_url": "<string>",
"text_hosted_url": "<string>",
"links_on_page": [
"<string>"
],
"page_metadata": {
"status_code": 123,
"title": "<string>"
}
},
"credits_consumed": 123,
"cost_usd": 123
}{
"id": "error_x2nmu5bqn6",
"object": "error",
"created": 1777923912,
"metadata": {},
"error": {
"type": "invalid_request_error",
"code": "dns_resolution_failed",
"message": "The URL contains a typo, or the domain does not exist."
}
}{
"id": "error_ogeb6rik8c",
"object": "error",
"created": 1777923969,
"url": "https://example.com",
"metadata": {},
"error": {
"type": "invalid_request_error",
"code": "tls_error",
"detail": "err_ssl_tlsv1_alert_internal_error",
"message": "The website closed or rejected the TLS handshake. The server may be misconfigured or use an unsupported SSL/TLS version."
}
}{
"id": "error_qat3d1amjt",
"object": "error",
"created": 1777923969,
"url": "https://example.com",
"metadata": {},
"error": {
"type": "request_timeout",
"code": "scrape_poll_timeout",
"message": "Request timed out while waiting for scrape result. The page may be slow, blocked for our fetchers, or temporarily unavailable."
}
}Abschürfungen
Scrape erstellen
Scrape eine URL mit bereitgestellter Konfiguration und erhalte den Inhalt.
POST
/
v1
/
scrapes
Eine Webseitenscrape initiieren
curl --request POST \
--url https://api.olostep.com/v1/scrapes \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url_to_scrape": "<string>",
"wait_before_scraping": 123,
"formats": [],
"actions": [
{
"type": "wait",
"milliseconds": 1
}
],
"country": "<string>",
"remove_images": false,
"remove_class_names": [
"<string>"
],
"llm_extract": {
"schema": {}
},
"links_on_page": {
"query_to_order_links_by": "<string>",
"include_links": [
"<string>"
],
"exclude_links": [
"<string>"
]
},
"screen_size": {
"screen_width": 123,
"screen_height": 123
},
"screenshot": {
"full_page": true
},
"metadata": {},
"max_age": 0
}
'import requests
url = "https://api.olostep.com/v1/scrapes"
payload = {
"url_to_scrape": "<string>",
"wait_before_scraping": 123,
"formats": [],
"actions": [
{
"type": "wait",
"milliseconds": 1
}
],
"country": "<string>",
"remove_images": False,
"remove_class_names": ["<string>"],
"llm_extract": { "schema": {} },
"links_on_page": {
"query_to_order_links_by": "<string>",
"include_links": ["<string>"],
"exclude_links": ["<string>"]
},
"screen_size": {
"screen_width": 123,
"screen_height": 123
},
"screenshot": { "full_page": True },
"metadata": {},
"max_age": 0
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
url_to_scrape: '<string>',
wait_before_scraping: 123,
formats: [],
actions: [{type: 'wait', milliseconds: 1}],
country: '<string>',
remove_images: false,
remove_class_names: ['<string>'],
llm_extract: {schema: {}},
links_on_page: {
query_to_order_links_by: '<string>',
include_links: ['<string>'],
exclude_links: ['<string>']
},
screen_size: {screen_width: 123, screen_height: 123},
screenshot: {full_page: true},
metadata: {},
max_age: 0
})
};
fetch('https://api.olostep.com/v1/scrapes', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.olostep.com/v1/scrapes",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url_to_scrape' => '<string>',
'wait_before_scraping' => 123,
'formats' => [
],
'actions' => [
[
'type' => 'wait',
'milliseconds' => 1
]
],
'country' => '<string>',
'remove_images' => false,
'remove_class_names' => [
'<string>'
],
'llm_extract' => [
'schema' => [
]
],
'links_on_page' => [
'query_to_order_links_by' => '<string>',
'include_links' => [
'<string>'
],
'exclude_links' => [
'<string>'
]
],
'screen_size' => [
'screen_width' => 123,
'screen_height' => 123
],
'screenshot' => [
'full_page' => true
],
'metadata' => [
],
'max_age' => 0
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.olostep.com/v1/scrapes"
payload := strings.NewReader("{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.olostep.com/v1/scrapes")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.olostep.com/v1/scrapes")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url_to_scrape\": \"<string>\",\n \"wait_before_scraping\": 123,\n \"formats\": [],\n \"actions\": [\n {\n \"type\": \"wait\",\n \"milliseconds\": 1\n }\n ],\n \"country\": \"<string>\",\n \"remove_images\": false,\n \"remove_class_names\": [\n \"<string>\"\n ],\n \"llm_extract\": {\n \"schema\": {}\n },\n \"links_on_page\": {\n \"query_to_order_links_by\": \"<string>\",\n \"include_links\": [\n \"<string>\"\n ],\n \"exclude_links\": [\n \"<string>\"\n ]\n },\n \"screen_size\": {\n \"screen_width\": 123,\n \"screen_height\": 123\n },\n \"screenshot\": {\n \"full_page\": true\n },\n \"metadata\": {},\n \"max_age\": 0\n}"
response = http.request(request)
puts response.read_body{
"id": "<string>",
"object": "<string>",
"created": 123,
"metadata": {},
"url_to_scrape": "<string>",
"result": {
"html_content": "<string>",
"markdown_content": "<string>",
"text_content": "<string>",
"json_content": "<string>",
"screenshot_hosted_url": "<string>",
"html_hosted_url": "<string>",
"markdown_hosted_url": "<string>",
"text_hosted_url": "<string>",
"links_on_page": [
"<string>"
],
"page_metadata": {
"status_code": 123,
"title": "<string>"
}
},
"credits_consumed": 123,
"cost_usd": 123
}{
"id": "error_x2nmu5bqn6",
"object": "error",
"created": 1777923912,
"metadata": {},
"error": {
"type": "invalid_request_error",
"code": "dns_resolution_failed",
"message": "The URL contains a typo, or the domain does not exist."
}
}{
"id": "error_ogeb6rik8c",
"object": "error",
"created": 1777923969,
"url": "https://example.com",
"metadata": {},
"error": {
"type": "invalid_request_error",
"code": "tls_error",
"detail": "err_ssl_tlsv1_alert_internal_error",
"message": "The website closed or rejected the TLS handshake. The server may be misconfigured or use an unsupported SSL/TLS version."
}
}{
"id": "error_qat3d1amjt",
"object": "error",
"created": 1777923969,
"url": "https://example.com",
"metadata": {},
"error": {
"type": "request_timeout",
"code": "scrape_poll_timeout",
"message": "Request timed out while waiting for scrape result. The page may be slow, blocked for our fetchers, or temporarily unavailable."
}
}Optionales Caching: Übergebe
max_age (in Sekunden), um einen kürzlich durchgeführten Scrape mit denselben Parametern wiederzuverwenden, anstatt die Seite erneut abzurufen. Standardmäßig ist 0 (immer frisch). Im Dashboard-Playground beträgt der Standardwert 24 Stunden. Siehe Caching für Details.Autorisierungen
Bearer-Authentifizierungsheader in der Form Bearer , wobei dein Authentifizierungstoken ist.
Body
application/json
Die URL, von der aus das Scraping gestartet werden soll.
Wartezeit in Millisekunden, bevor das Scraping beginnt.
Formate, in denen du den Inhalt haben möchtest.
Verfügbare Optionen:
html, markdown, text, json, raw_pdf, screenshot Option, bestimmte CSS-Selektoren aus dem Inhalt zu entfernen. Optional kannst du auch ein JSON-stringifiziertes Array von spezifischen Selektoren übergeben, die du entfernen möchtest. Die CSS-Selektoren, die entfernt werden, wenn diese Option auf Standard gesetzt ist, sind ['nav','footer','script','style','noscript','svg',[role=alert],[role=banner],[role=dialog],[role=alertdialog],[role=region][aria-label*=skip i],[aria-modal=true]]
Verfügbare Optionen:
default, none, array Aktionen, die auf der Seite ausgeführt werden sollen, bevor der Inhalt abgerufen wird.
- Warten
- Klicken
- Eingabe ausfüllen
- Scrollen
Show child attributes
Show child attributes
Wohnsitzland, von dem aus die Anfrage geladen wird. Unterstützte Werte sind: - US (Vereinigte Staaten) - CA (Kanada) - IT (Italien) - IN (Indien) - GB (England) - JP (Japan) - MX (Mexiko) - AU (Australien) - ID (Indonesien) - UA (VAE) - RU (Russland) - RANDOM Einige Operationen, wie das Scraping von Google Search und Google News, unterstützen alle Länder.
Gib den HTML-Transformer an, der verwendet werden soll, falls vorhanden. Die Mercury Parser-Bibliothek von Postlight wird verwendet, um Werbung und andere unerwünschte Inhalte aus dem gescrapten Inhalt zu entfernen.
Verfügbare Optionen:
postlight, none Option, Bilder aus dem gescrapten Inhalt zu entfernen. Standardmäßig auf false gesetzt.
Liste von Klassennamen, die aus dem Inhalt entfernt werden sollen.
Wenn du json als Format definierst, kannst du diesen Parameter verwenden, um den zu verwendenden Parser anzugeben. Parser sind nützlich, um strukturierten Inhalt aus Webseiten zu extrahieren. Olostep hat einige Parser für die gängigsten Webseiten eingebaut, und du kannst auch deine eigenen Parser erstellen.
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Mit dieser Option kannst du alle Links erhalten, die auf der Seite vorhanden sind, die du scrapst. Links werden immer als absolute URLs zurückgegeben.
Show child attributes
Show child attributes
Konfiguration für Bildschirmgröße. Voreingestellte Abmessungen sind über screen_type verfügbar: desktop (1920x1080), mobile (414x896) oder default (768x1024).
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Benutzerdefinierte Metadaten. Noch nicht unterstützt.
Maximales akzeptables Alter des zwischengespeicherten Inhalts in Sekunden. Wenn ein passender Scrape bereits existiert und neuer als max_age Sekunden ist, gibt Olostep das gespeicherte Ergebnis zurück, anstatt einen neuen Browser-Scrape zu starten. Standardmäßig 0 (immer frisch scrapen). Im Dashboard-Playground ist der Standard 86400 (24 Stunden). Der maximal erlaubte Wert ist 604800 (7 Tage). Siehe den Abschnitt Caching in den Scrapes-Feature-Dokumenten für Details.
Erforderlicher Bereich:
x >= 0Antwort
Erfolgreiche Antwort mit den Details zur Scrape-Initiierung.
Scrape-ID
Die Art des Objekts. "scrape" für diesen Endpunkt.
Erstellte Epoche
Benutzerdefinierte Metadaten.
Die URL, die gescrapt wurde.
Show child attributes
Show child attributes
Anzahl der durch diese Anfrage verbrauchten Credits. Wird nach Abschluss der Ausführung ausgefüllt. Credits sind die Grundlage für die Abrechnung.
Geschätzte Kosten in USD für diese Anfrage. Wird nach Abschluss der Ausführung ausgefüllt. Berechnet aus den verbrauchten Credits und deinem Tarif — 99% genau, aber credits_consumed ist der maßgebliche Wert.
War diese Seite hilfreich?