{ "openapi": "3.0.3", "info": { "title": "Batches API", "version": "1.0.0" }, "servers": [ { "url": "https://api.olostep.com" } ], "paths": { "/v1/test-batch": { "post": { "summary": "Test batch", "description": "Test batch", "security": [ { "Authorization": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "properties": { "items": { "type": "array", "items": { "type": "object", "properties": { "custom_id": { "type": "string", "description": "An internal unique identifier for the url." }, "url": { "type": "string", "format": "uri", "description": "URL of the item." } }, "required": [ "custom_id", "url" ] }, "description": "Array of items to be processed in the batch." } }, "required": [ "items" ] } } } }, "responses": { "200": { "description": "Batch started successfully.", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "description": "Batch ID" } } } } } } } } }, "/v1/batches/recent": { "get": { "summary": "List recent batches", "description": "Fetches the list of recent batches for the authenticated user.", "security": [ { "Authorization": [] } ], "parameters": [ { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer", "default": 50, "maximum": 100 }, "description": "Maximum number of batches to return (default 50, max 100)." }, { "name": "source", "in": "query", "required": false, "schema": { "type": "string" }, "description": "Filter by source (optional)." } ], "responses": { "200": { "description": "Successful response with list of recent batches.", "content": { "application/json": { "schema": { "type": "object", "properties": { "object": { "type": "string", "enum": [ "list" ] }, "data": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string" }, "object": { "type": "string" }, "status": { "type": "string" }, "created": { "type": "number" }, "total_urls": { "type": "number" }, "completed_urls": { "type": "number" } } } } } } } } }, "401": { "description": "Authentication credentials are missing or invalid. See [Unauthorized](/api-reference/errors/unauthorized) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_api_key", "type": "https://docs.olostep.com/api-reference/errors/unauthorized", "status": 401, "title": "Unauthorized", "detail": "Your API key is invalid. Try double-checking it or reaching out to info@olostep.com if you're facing issues.", "created": 1704067200, "metadata": {} } } } }, "500": { "description": "Internal server error. See [Internal Error](/api-reference/errors/internal_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "internal_server_error", "type": "https://docs.olostep.com/api-reference/errors/internal_error", "status": 500, "title": "Internal Server Error", "detail": "An unexpected error occurred", "created": 1704067200, "metadata": {} } } } } } } }, "/v1/batches": { "post": { "summary": "Start a new batch", "description": "Initiates a new batch process with the specified parameters.", "security": [ { "Authorization": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "properties": { "items": { "type": "array", "items": { "type": "object", "properties": { "custom_id": { "type": "string", "description": "An internal unique identifier for the url." }, "url": { "type": "string", "format": "uri", "description": "URL of the item." }, "metadata": { "allOf": [ { "$ref": "#/components/schemas/Metadata" } ], "description": "Item-level metadata. Attach key-value pairs to individual items for tracking, filtering, or correlation with your internal systems." } }, "required": [ "custom_id", "url" ] }, "description": "Array of items to be processed in the batch." }, "country": { "type": "string", "description": "Country for the batch execution. Provide in ISO 3166-1 alpha-2 codes like US(USA), IN(India), etc" }, "parser": { "type": "object", "properties": { "id": { "type": "string", "description": "Parser to be used for the batch." } }, "required": [ "id" ], "description": "You can use this parameter to specify the parser to use. Parsers are useful to extract structured content from web pages. Olostep has a few parsers built in for most common web pages, and you can also create your own parsers." }, "links_on_page": { "type": "object", "properties": { "include_links": { "type": "array", "items": { "type": "string" }, "description": "Filter extracted links using glob patterns. Patterns match against the link's URL path. Note: a single `*` does not cross `/`, so \"/blog/*\" matches \"/blog/post-1\" but NOT the index \"/blog\" itself (or \"/blog?tag=x\", since query strings are not part of the path). To include the index too, use \"/blog*\" or \"{/blog,/blog/**}\"." }, "exclude_links": { "type": "array", "items": { "type": "string" }, "description": "Filter extracted links using glob patterns. Patterns match against the link's URL path. Note: a single `*` does not cross `/`, so \"/blog/*\" matches \"/blog/post-1\" but NOT the index \"/blog\" itself." } }, "description": "Get all the links present on each page in the batch. Links are always returned as absolute URLs." }, "metadata": { "$ref": "#/components/schemas/Metadata" }, "webhook": { "type": "string", "format": "uri", "description": "HTTPS URL to receive a POST request when the batch completes. Must be a publicly accessible URL using `http://` or `https://` protocol. Cannot point to localhost or private IP addresses. See [Webhooks](/api-reference/common/webhooks) for payload format and retry behavior." } }, "required": [ "items" ] }, "example": { "items": [ { "custom_id": "product-123", "url": "https://example.com/product/123", "metadata": { "source": "catalog_sync", "priority": "high" } }, { "custom_id": "product-456", "url": "https://example.com/product/456" } ], "country": "US", "metadata": { "batch_name": "Q1 Product Sync", "initiated_by": "automation" } } } } }, "responses": { "200": { "description": "Batch started successfully.", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "description": "Batch ID" }, "object": { "type": "string", "description": "The kind of object. \"batch\" for this endpoint." }, "status": { "type": "string", "description": "`in_progress` or `completed`" }, "created": { "type": "number", "description": "Created epoch" }, "total_urls": { "type": "number", "description": "Count of URLs in the batch" }, "completed_urls": { "type": "number", "description": "Count of completed URLs" }, "parser": { "type": "string" }, "country": { "type": "string" }, "metadata": { "$ref": "#/components/schemas/Metadata" }, "webhook": { "type": "string", "description": "Webhook URL to receive completion notification" }, "credits_consumed": { "type": "integer", "nullable": true, "description": "Number of credits consumed by this request. Populated after execution completes. Credits are the source of truth for billing." }, "cost_usd": { "type": "number", "nullable": true, "description": "Estimated cost in USD for this request. Populated after execution completes. Calculated from credits consumed and your plan rate \u2014 99% accurate, but credits_consumed is the authoritative value." } } }, "example": { "id": "batch_abc123def456", "object": "batch", "status": "in_progress", "created": 1704067200, "total_urls": 2, "completed_urls": 0, "country": "US", "metadata": { "batch_name": "Q1 Product Sync", "initiated_by": "automation" } } } } }, "400": { "description": "Bad request due to incorrect or missing parameters. See [Bad Request](/api-reference/errors/bad_request) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "validation_error", "type": "https://docs.olostep.com/api-reference/errors/bad_request", "status": 400, "title": "Bad Request", "detail": "The 'items' array is required and must contain at least one item.", "created": 1704067200, "metadata": {} } } } }, "401": { "description": "Authentication credentials are missing or invalid. See [Unauthorized](/api-reference/errors/unauthorized) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_api_key", "type": "https://docs.olostep.com/api-reference/errors/unauthorized", "status": 401, "title": "Unauthorized", "detail": "Your API key is invalid. Try double-checking it or reaching out to info@olostep.com if you're facing issues.", "created": 1704067200, "metadata": {} } } } }, "402": { "description": "Payment required - credits exhausted. See [Payment Required](/api-reference/errors/payment_required) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "credits_exhausted", "type": "https://docs.olostep.com/api-reference/errors/payment_required", "status": 402, "title": "Payment Required", "detail": "You have consumed all available credits. Please upgrade your plan from the dashboard: https://www.olostep.com/auth/", "created": 1704067200, "metadata": {} } } } }, "403": { "description": "Forbidden - access denied to this feature. See [Forbidden](/api-reference/errors/forbidden) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "access_denied", "type": "https://docs.olostep.com/api-reference/errors/forbidden", "status": 403, "title": "Forbidden", "detail": "You don't have access to this feature. Please reach out to info@olostep.com to get approved", "created": 1704067200, "metadata": {} } } } }, "409": { "description": "Idempotency conflict - a request with this key is in progress. See [Idempotency Error](/api-reference/errors/idempotency_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "idempotency_key_in_progress", "type": "https://docs.olostep.com/api-reference/errors/idempotency_error", "status": 409, "title": "Idempotency Error", "detail": "A request with this idempotency key is currently being processed. Please wait and retry.", "created": 1704067200, "metadata": {} } } } }, "422": { "description": "Unprocessable entity - business rule violation. See [Unprocessable Entity](/api-reference/errors/unprocessable_entity) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "idempotency_key_reuse", "type": "https://docs.olostep.com/api-reference/errors/unprocessable_entity", "status": 422, "title": "Unprocessable Entity", "detail": "A request with this idempotency key was already made with different parameters. Idempotency keys must be unique per request.", "created": 1704067200, "metadata": {} } } } }, "500": { "description": "Internal server error. See [Internal Error](/api-reference/errors/internal_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "internal_server_error", "type": "https://docs.olostep.com/api-reference/errors/internal_error", "status": 500, "title": "Internal Server Error", "detail": "An unexpected error occurred", "created": 1704067200, "metadata": {} } } } } } } }, "/v1/batches/{batch_id}": { "get": { "summary": "Retrieve batch information", "description": "Fetches information about a specific batch using its `batch_id`.", "security": [ { "Authorization": [] } ], "parameters": [ { "name": "batch_id", "in": "path", "required": true, "schema": { "type": "string" }, "description": "The ID of the batch to retrieve information for." } ], "responses": { "200": { "description": "Successful response with batch information.", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "description": "Batch ID" }, "object": { "type": "string", "description": "The kind of object. \"batch\" for this endpoint." }, "status": { "type": "string", "description": "`in_progress` or `completed`" }, "created": { "type": "number", "description": "Created epoch" }, "total_urls": { "type": "number", "description": "Count of URLs in the batch" }, "completed_urls": { "type": "number", "description": "Count of completed URLs" }, "parser": { "type": "string" }, "country": { "type": "string" }, "metadata": { "$ref": "#/components/schemas/Metadata" }, "webhook": { "type": "string", "description": "Webhook URL to receive completion notification" }, "credits_consumed": { "type": "integer", "nullable": true, "description": "Number of credits consumed by this request. Populated after execution completes. Credits are the source of truth for billing." }, "cost_usd": { "type": "number", "nullable": true, "description": "Estimated cost in USD for this request. Populated after execution completes. Calculated from credits consumed and your plan rate \u2014 99% accurate, but credits_consumed is the authoritative value." } } } } } }, "401": { "description": "Authentication credentials are missing or invalid. See [Unauthorized](/api-reference/errors/unauthorized) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_api_key", "type": "https://docs.olostep.com/api-reference/errors/unauthorized", "status": 401, "title": "Unauthorized", "detail": "Your API key is invalid", "created": 1704067200, "metadata": {} } } } }, "404": { "description": "Batch not found for the provided ID. See [Not Found](/api-reference/errors/not_found) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "batch_not_found", "type": "https://docs.olostep.com/api-reference/errors/not_found", "status": 404, "title": "Not Found", "detail": "Batch not found", "created": 1704067200, "metadata": {} } } } }, "500": { "description": "Internal server error. See [Internal Error](/api-reference/errors/internal_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "internal_server_error", "type": "https://docs.olostep.com/api-reference/errors/internal_error", "status": 500, "title": "Internal Server Error", "detail": "An unexpected error occurred", "created": 1704067200, "metadata": {} } } } } } }, "patch": { "summary": "Update batch metadata", "description": "Updates the metadata for a specific batch. Only the `metadata` field can be updated. Follows Stripe's merge semantics: new keys are added, existing keys are updated, and keys set to empty string are deleted.", "security": [ { "Authorization": [] } ], "parameters": [ { "name": "batch_id", "in": "path", "required": true, "schema": { "type": "string" }, "description": "The ID of the batch to update." } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "properties": { "metadata": { "$ref": "#/components/schemas/Metadata" } }, "required": [ "metadata" ] }, "example": { "metadata": { "status": "reviewed", "reviewer": "jane@example.com" } } } } }, "responses": { "200": { "description": "Batch metadata updated successfully.", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "description": "Batch ID" }, "object": { "type": "string", "description": "The kind of object. \"batch\" for this endpoint." }, "status": { "type": "string", "description": "`in_progress` or `completed`" }, "created": { "type": "number", "description": "Created epoch" }, "total_urls": { "type": "number", "description": "Count of URLs in the batch" }, "completed_urls": { "type": "number", "description": "Count of completed URLs" }, "metadata": { "$ref": "#/components/schemas/Metadata" } } }, "example": { "id": "batch_abc123def456", "object": "batch", "status": "completed", "created": 1704067200, "total_urls": 10, "completed_urls": 10, "metadata": { "batch_name": "Q1 Product Sync", "status": "reviewed", "reviewer": "jane@example.com" } } } } }, "400": { "description": "Bad request - missing or invalid metadata. See [Bad Request](/api-reference/errors/bad_request) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_metadata", "type": "https://docs.olostep.com/api-reference/errors/bad_request", "status": 400, "title": "Bad Request", "detail": "No metadata field provided. Only metadata can be updated.", "created": 1704067200, "metadata": {} } } } }, "401": { "description": "Authentication credentials are missing or invalid. See [Unauthorized](/api-reference/errors/unauthorized) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_api_key", "type": "https://docs.olostep.com/api-reference/errors/unauthorized", "status": 401, "title": "Unauthorized", "detail": "Your API key is invalid", "created": 1704067200, "metadata": {} } } } }, "404": { "description": "Batch not found for the provided ID. See [Not Found](/api-reference/errors/not_found) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "batch_not_found", "type": "https://docs.olostep.com/api-reference/errors/not_found", "status": 404, "title": "Not Found", "detail": "Batch not found", "created": 1704067200, "metadata": {} } } } }, "500": { "description": "Internal server error. See [Internal Error](/api-reference/errors/internal_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "internal_server_error", "type": "https://docs.olostep.com/api-reference/errors/internal_error", "status": 500, "title": "Internal Server Error", "detail": "An unexpected error occurred", "created": 1704067200, "metadata": {} } } } } } } }, "/v1/batches/{batch_id}/items": { "get": { "summary": "Retrieve completed or failed items list optionally with content for a specific batch", "description": "Fetches the list of items and content that as per the requested status for a specific batch ID.", "security": [ { "Authorization": [] } ], "parameters": [ { "name": "batch_id", "in": "path", "required": true, "schema": { "type": "string" }, "description": "The ID of the batch to retrieve items for." }, { "name": "status", "in": "query", "required": false, "schema": { "type": "string", "enum": [ "completed", "failed" ] }, "description": "Status of the URLs to retrieve (completed or failed)." }, { "name": "cursor", "in": "query", "required": false, "schema": { "type": "integer" }, "description": "Pagination cursor. Omit this parameter on the first request. For subsequent requests, use the `cursor` value from the previous response. See [Pagination](/api-reference/common/pagination) for details." }, { "name": "limit", "in": "query", "required": false, "schema": { "type": "integer" }, "description": "Number of results to return (recommended 10-50)." } ], "responses": { "200": { "description": "Successful response with batch items.", "content": { "application/json": { "schema": { "type": "object", "properties": { "batch_id": { "type": "string" }, "object": { "type": "string" }, "status": { "type": "string" }, "items": { "type": "array", "items": { "type": "object", "properties": { "custom_id": { "type": "string" }, "retrieve_id": { "type": "string" }, "url": { "type": "string" }, "metadata": { "allOf": [ { "$ref": "#/components/schemas/Metadata" } ], "description": "Item-level metadata if provided during batch creation." }, "links_on_page": { "type": "array", "items": { "type": "string" } } } } }, "items_count": { "type": "integer" }, "cursor": { "type": "integer", "description": "Pagination cursor. Present when there are more results available. Use this value in the next request's `cursor` parameter. See [Pagination](/api-reference/common/pagination) for details." } } }, "example": { "batch_id": "batch_abc123def456", "object": "list", "status": "completed", "items": [ { "custom_id": "product-123", "retrieve_id": "ret_xyz789", "url": "https://example.com/product/123", "metadata": { "source": "catalog_sync", "priority": "high" } }, { "custom_id": "product-456", "retrieve_id": "ret_abc456", "url": "https://example.com/product/456" } ], "items_count": 2 } } } }, "400": { "description": "Bad request due to incorrect parameters. See [Bad Request](/api-reference/errors/bad_request) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "validation_error", "type": "https://docs.olostep.com/api-reference/errors/bad_request", "status": 400, "title": "Bad Request", "detail": "Status must be either 'completed' or 'failed'", "created": 1704067200, "metadata": {} } } } }, "401": { "description": "Authentication credentials are missing or invalid. See [Unauthorized](/api-reference/errors/unauthorized) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "invalid_api_key", "type": "https://docs.olostep.com/api-reference/errors/unauthorized", "status": 401, "title": "Unauthorized", "detail": "Your API key is invalid", "created": 1704067200, "metadata": {} } } } }, "404": { "description": "Batch not found for the provided ID. See [Not Found](/api-reference/errors/not_found) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "batch_not_found", "type": "https://docs.olostep.com/api-reference/errors/not_found", "status": 404, "title": "Not Found", "detail": "Batch not found", "created": 1704067200, "metadata": {} } } } }, "500": { "description": "Internal server error. See [Internal Error](/api-reference/errors/internal_error) for details.", "content": { "application/problem+json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "id": "error_abc123", "object": "error", "code": "internal_server_error", "type": "https://docs.olostep.com/api-reference/errors/internal_error", "status": 500, "title": "Internal Server Error", "detail": "An unexpected error occurred", "created": 1704067200, "metadata": {} } } } } } } } }, "components": { "securitySchemes": { "Authorization": { "type": "http", "scheme": "bearer", "description": "Bearer authentication header of the form Bearer , where is your auth token." } }, "schemas": { "Error": { "type": "object", "description": "RFC 7807 Problem Details error response", "properties": { "id": { "type": "string", "description": "Unique error identifier" }, "object": { "type": "string", "enum": [ "error" ], "description": "Always 'error'" }, "code": { "type": "string", "description": "Machine-readable error code" }, "type": { "type": "string", "format": "uri", "description": "URI reference identifying the problem type" }, "status": { "type": "integer", "description": "HTTP status code" }, "title": { "type": "string", "description": "Short, human-readable summary" }, "detail": { "type": "string", "description": "Human-readable explanation" }, "created": { "type": "integer", "description": "Unix timestamp" }, "metadata": { "$ref": "#/components/schemas/Metadata" }, "errors": { "type": "array", "description": "Optional array of additional error details", "items": {} } }, "required": [ "id", "object", "code", "type", "status", "title", "detail", "created" ] }, "Metadata": { "type": "object", "description": "Set of key-value pairs for storing additional information about an object. Follows Stripe's approach with validation rules: max 50 keys, key max 40 characters (no square brackets), value max 500 characters, all values stored as strings.", "additionalProperties": { "type": "string", "maxLength": 500, "description": "Metadata value (max 500 characters). Numbers and booleans are automatically converted to strings." }, "maxProperties": 50, "example": { "order_id": "12345", "customer_name": "John Doe", "priority": "high", "processed": "true" }, "x-validation-rules": { "max_keys": 50, "key_max_length": 40, "key_forbidden_chars": [ "[", "]" ], "value_max_length": 500, "value_types": [ "string", "number (coerced)", "boolean (coerced)" ] } } } } }