{
    "openapi": "3.0.3",
    "info": {
        "title": "AirOps Batches API",
        "version": "1.0.0",
        "description": "Internal API for AirOps-specific batch scraping with automated LLM enrichment for page metadata extraction."
    },
    "servers": [
        {
            "url": "https://api.olostep.com"
        }
    ],
    "paths": {
        "/v1/batches-airops": {
            "post": {
                "operationId": "createAirOpsBatch",
                "summary": "Create AirOps Batch",
                "description": "Starts a batch process that combines scraping with LLM-based enrichment for page metadata extraction. The workflow: scrape URLs → extract metadata → gap analysis → LLM enrichment for missing fields.",
                "security": [
                    {
                        "Authorization": []
                    }
                ],
                "requestBody": {
                    "required": true,
                    "content": {
                        "application/json": {
                            "schema": {
                                "$ref": "#/components/schemas/CreateBatchRequest"
                            },
                            "example": {
                                "items": [
                                    { "url": "https://example.com/blog/post-1", "custom_id": "post-1" },
                                    { "url": "https://example.com/blog/post-2", "custom_id": "post-2" }
                                ],
                                "brands": ["Acme Corp", "TechBrand"],
                                "country": "US",
                                "webhook": "https://your-server.com/webhook"
                            }
                        }
                    }
                },
                "responses": {
                    "200": {
                        "description": "Batch created successfully.",
                        "content": {
                            "application/json": {
                                "schema": {
                                    "$ref": "#/components/schemas/CreateBatchResponse"
                                },
                                "example": {
                                    "id": "batch_abc123xyz",
                                    "object": "batch_airops",
                                    "created": 1705315200000,
                                    "total_urls": 2,
                                    "status": {
                                        "overall": "in_progress",
                                        "scraping": {
                                            "state": "started",
                                            "submitted": 2,
                                            "valid": 2,
                                            "succeeded": 0,
                                            "failed": 0,
                                            "started_at": 1705315200000,
                                            "completed_at": null,
                                            "error": null
                                        },
                                        "analyzing": {
                                            "state": "pending",
                                            "items_analyzed": 0,
                                            "items_needing_enrichment": 0,
                                            "items_complete": 0,
                                            "fields_to_enrich": null,
                                            "started_at": null,
                                            "completed_at": null
                                        },
                                        "enriching": {
                                            "state": "pending",
                                            "items_to_enrich": 0,
                                            "items_enriched": 0,
                                            "items_failed": 0,
                                            "llm_batch_id": null,
                                            "started_at": null,
                                            "completed_at": null,
                                            "error": null
                                        }
                                    }
                                }
                            }
                        }
                    },
                    "400": {
                        "description": "Validation error.",
                        "content": {
                            "application/json": {
                                "schema": {
                                    "$ref": "#/components/schemas/ValidationError"
                                }
                            }
                        }
                    },
                    "401": {
                        "description": "Authentication credentials are missing or invalid.",
                        "content": {
                            "application/json": {
                                "schema": { "$ref": "#/components/schemas/Error" }
                            }
                        }
                    },
                    "500": {
                        "description": "Internal server error.",
                        "content": {
                            "application/json": {
                                "schema": { "$ref": "#/components/schemas/Error" }
                            }
                        }
                    }
                }
            }
        },
        "/v1/batches-airops/{batch_id}": {
            "get": {
                "operationId": "getAirOpsBatch",
                "summary": "Get AirOps Batch Status",
                "description": "Retrieve the current status and configuration of an AirOps batch.",
                "security": [
                    {
                        "Authorization": []
                    }
                ],
                "parameters": [
                    {
                        "name": "batch_id",
                        "in": "path",
                        "required": true,
                        "schema": { "type": "string" },
                        "description": "The ID of the batch to retrieve.",
                        "example": "batch_abc123xyz"
                    }
                ],
                "responses": {
                    "200": {
                        "description": "Batch status retrieved successfully.",
                        "content": {
                            "application/json": {
                                "schema": {
                                    "$ref": "#/components/schemas/BatchDetailResponse"
                                },
                                "example": {
                                    "id": "batch_abc123xyz",
                                    "object": "batch",
                                    "created_at": 1705315200000,
                                    "total_urls": 50,
                                    "status": "completed",
                                    "batch_mode": true,
                                    "scraping": {
                                        "state": "completed",
                                        "submitted": 50,
                                        "valid": 50,
                                        "succeeded": 48,
                                        "failed": 2,
                                        "started_at": 1705315200000,
                                        "completed_at": 1705315500000,
                                        "error": null
                                    },
                                    "analyzing": {
                                        "state": "completed",
                                        "items_analyzed": 48,
                                        "items_needing_enrichment": 30,
                                        "items_complete": 18,
                                        "started_at": 1705315500000,
                                        "completed_at": 1705315502000
                                    },
                                    "enriching": {
                                        "state": "completed",
                                        "items_to_enrich": 30,
                                        "items_enriched": 30,
                                        "items_failed": 0,
                                        "llm_batch_id": "batch_xyz789",
                                        "batch_mode": true,
                                        "started_at": 1705315502000,
                                        "completed_at": 1705316800000,
                                        "error": null
                                    }
                                }
                            }
                        }
                    },
                    "404": {
                        "description": "Batch not found.",
                        "content": {
                            "application/json": {
                                "schema": { "$ref": "#/components/schemas/Error" }
                            }
                        }
                    }
                }
            }
        },
        "/v1/batches-airops/{batch_id}/items": {
            "get": {
                "operationId": "getAirOpsBatchItems",
                "summary": "Get AirOps Batch Items",
                "description": "Retrieve all items in the batch with their parser and enriched data.",
                "security": [
                    {
                        "Authorization": []
                    }
                ],
                "parameters": [
                    {
                        "name": "batch_id",
                        "in": "path",
                        "required": true,
                        "schema": { "type": "string" },
                        "description": "The ID of the batch to retrieve items for.",
                        "example": "batch_abc123xyz"
                    }
                ],
                "responses": {
                    "200": {
                        "description": "Batch items retrieved successfully.",
                        "content": {
                            "application/json": {
                                "schema": {
                                    "$ref": "#/components/schemas/ListBatchItemsResponse"
                                },
                                "example": {
                                    "batch_id": "batch_abc123xyz",
                                    "completed_at": "2024-01-15T12:30:00.000Z",
                                    "status": "completed",
                                    "counts": {
                                        "total": 3,
                                        "succeeded": 2,
                                        "failed": 1
                                    },
                                    "items": [
                                        {
                                            "custom_id": "post-1",
                                            "url": "https://example.com/blog/post-1",
                                            "status": "completed",
                                            "page_type": "informational_article",
                                            "author": "John Doe",
                                            "date_published": "2024-01-15",
                                            "date_modified": null,
                                            "brand_mentions": ["Acme Corp"]
                                        },
                                        {
                                            "custom_id": "post-2",
                                            "url": "https://example.com/pricing",
                                            "status": "completed",
                                            "page_type": "pricing_page",
                                            "author": null,
                                            "date_published": null,
                                            "date_modified": null,
                                            "brand_mentions": []
                                        },
                                        {
                                            "custom_id": "post-3",
                                            "url": "https://example.com/broken",
                                            "status": "failed",
                                            "failed_stage": "scraping",
                                            "error": {
                                                "code": "SCRAPE_FAILED",
                                                "message": "URL failed to scrape"
                                            }
                                        }
                                    ]
                                }
                            }
                        }
                    },
                    "404": {
                        "description": "Batch not found.",
                        "content": {
                            "application/json": {
                                "schema": { "$ref": "#/components/schemas/Error" }
                            }
                        }
                    }
                }
            }
        }
    },
    "components": {
        "securitySchemes": {
            "Authorization": {
                "type": "http",
                "scheme": "bearer",
                "description": "Bearer authentication header of the form Bearer <token>, where <token> is your auth token."
            }
        },
        "schemas": {
            "CreateBatchRequest": {
                "type": "object",
                "required": ["items"],
                "properties": {
                    "items": {
                        "type": "array",
                        "minItems": 1,
                        "maxItems": 10000,
                        "items": {
                            "$ref": "#/components/schemas/BatchItem"
                        },
                        "description": "Array of items to process."
                    },
                    "brands": {
                        "type": "array",
                        "items": {
                            "type": "string",
                            "minLength": 1,
                            "maxLength": 100
                        },
                        "default": [],
                        "description": "Brand names to look for in page content. The LLM will extract mentions of these brands from each page."
                    },
                    "country": {
                        "type": "string",
                        "default": "RANDOM",
                        "description": "Country for geo-targeted scraping. Use ISO 3166-1 alpha-2 codes (US, GB, DE, etc.) or 'RANDOM' for automatic selection."
                    },
                    "webhook": {
                        "type": "string",
                        "format": "uri",
                        "nullable": true,
                        "description": "URL to receive webhook notifications for batch lifecycle events."
                    },
                    "metadata": {
                        "$ref": "#/components/schemas/Metadata"
                    }
                }
            },
            "BatchItem": {
                "type": "object",
                "required": ["url", "custom_id"],
                "properties": {
                    "url": {
                        "type": "string",
                        "format": "uri",
                        "description": "The URL to scrape."
                    },
                    "custom_id": {
                        "type": "string",
                        "minLength": 1,
                        "maxLength": 255,
                        "description": "Your unique identifier for this item."
                    }
                }
            },
            "CreateBatchResponse": {
                "type": "object",
                "properties": {
                    "id": {
                        "type": "string",
                        "description": "Unique batch identifier."
                    },
                    "object": {
                        "type": "string",
                        "enum": ["batch_airops"],
                        "description": "Object type identifier."
                    },
                    "created": {
                        "type": "integer",
                        "description": "Unix timestamp (milliseconds) when the batch was created."
                    },
                    "total_urls": {
                        "type": "integer",
                        "description": "Total number of URLs in the batch."
                    },
                    "status": {
                        "$ref": "#/components/schemas/WorkflowStatus"
                    }
                }
            },
            "BatchDetailResponse": {
                "type": "object",
                "properties": {
                    "id": { "type": "string" },
                    "object": { "type": "string", "enum": ["batch"] },
                    "created_at": { "type": "integer", "description": "Unix timestamp (ms)" },
                    "total_urls": { "type": "integer" },
                    "status": {
                        "type": "string",
                        "enum": ["pending", "in_progress", "completed", "failed"],
                        "description": "Overall workflow status."
                    },
                    "batch_mode": { "type": "boolean", "description": "Whether async LLM batch mode is enabled" },
                    "scraping": { "$ref": "#/components/schemas/ScrapingStatus" },
                    "analyzing": { "$ref": "#/components/schemas/AnalyzingStatus" },
                    "enriching": { "$ref": "#/components/schemas/EnrichingStatus" }
                }
            },
            "ListBatchItemsResponse": {
                "type": "object",
                "properties": {
                    "batch_id": { "type": "string", "description": "The batch identifier." },
                    "completed_at": { "type": "string", "description": "ISO 8601 timestamp when batch completed." },
                    "status": { 
                        "type": "string", 
                        "enum": ["completed", "in_progress"],
                        "description": "Batch status."
                    },
                    "counts": {
                        "type": "object",
                        "properties": {
                            "total": { "type": "integer", "description": "Total items in batch." },
                            "succeeded": { "type": "integer", "description": "Successfully processed items." },
                            "failed": { "type": "integer", "description": "Failed items." }
                        }
                    },
                    "items": {
                        "type": "array",
                        "items": {
                            "$ref": "#/components/schemas/BatchItemResult"
                        }
                    },
                    "message": { 
                        "type": "string", 
                        "description": "Status message (present when batch is still processing)." 
                    }
                }
            },
            "BatchItemResult": {
                "type": "object",
                "description": "A processed item. Structure varies based on status.",
                "properties": {
                    "custom_id": { "type": "string", "description": "Your identifier from the request." },
                    "url": { "type": "string", "description": "The scraped URL." },
                    "status": { 
                        "type": "string", 
                        "enum": ["completed", "failed"],
                        "description": "Item processing status."
                    },
                    "page_type": {
                        "type": "string",
                        "nullable": true,
                        "enum": [
                            "homepage", "product_page", "collection_page", "pricing_page",
                            "informational_article", "documentation", "listicle_article",
                            "comparison_page", "support_article", "review_page", "forum_thread",
                            "social_media_post", "social_media_profile", "video_page", "news_article", "case_study",
                            "marketplace_listing", "landing_page", "deal_page", "job_posting", "other"
                        ],
                        "description": "Classified page type (present when status is completed)."
                    },
                    "author": { "type": "string", "nullable": true, "description": "Author of the content." },
                    "date_published": { "type": "string", "nullable": true, "description": "Publication date." },
                    "date_modified": { "type": "string", "nullable": true, "description": "Last modification date." },
                    "brand_mentions": { 
                        "type": "array", 
                        "items": { "type": "string" },
                        "description": "Brands found on the page."
                    },
                    "failed_stage": { 
                        "type": "string", 
                        "enum": ["scraping", "analyzing", "enriching"],
                        "description": "Stage where failure occurred (present when status is failed)."
                    },
                    "error": { 
                        "$ref": "#/components/schemas/StepError",
                        "description": "Error details (present when status is failed)."
                    }
                }
            },
            "EnrichedData": {
                "type": "object",
                "nullable": true,
                "description": "Final merged output combining parser and LLM extraction results.",
                "properties": {
                    "url": { "type": "string" },
                    "page_type": {
                        "type": "string",
                        "nullable": true,
                        "enum": [
                            "homepage", "product_page", "collection_page", "pricing_page",
                            "informational_article", "documentation", "listicle_article",
                            "comparison_page", "support_article", "review_page", "forum_thread",
                            "social_media_post", "social_media_profile", "video_page", "news_article", "case_study",
                            "marketplace_listing", "landing_page", "deal_page", "job_posting", "other"
                        ],
                        "description": "Classified page type."
                    },
                    "author": {
                        "type": "string",
                        "nullable": true,
                        "description": "Author of the content."
                    },
                    "date_published": {
                        "type": "string",
                        "nullable": true,
                        "description": "Publication date (ISO 8601 format when available)."
                    },
                    "date_modified": {
                        "type": "string",
                        "nullable": true,
                        "description": "Last modification date."
                    },
                    "brand_mentions": {
                        "type": "array",
                        "items": { "type": "string" },
                        "description": "Brands from your input list that were mentioned on the page."
                    },
                    "methods": {
                        "type": "object",
                        "description": "Extraction method used for each field.",
                        "additionalProperties": {
                            "type": "string",
                            "enum": ["classifier", "parser", "llm", "not_found"]
                        }
                    }
                }
            },
            "WorkflowStatus": {
                "type": "object",
                "description": "Unified status object showing all workflow phases.",
                "properties": {
                    "overall": {
                        "type": "string",
                        "enum": ["pending", "in_progress", "completed", "failed"],
                        "description": "Aggregate workflow state."
                    },
                    "scraping": { "$ref": "#/components/schemas/ScrapingStatus" },
                    "analyzing": { "$ref": "#/components/schemas/AnalyzingStatus" },
                    "enriching": { "$ref": "#/components/schemas/EnrichingStatus" }
                }
            },
            "ScrapingStatus": {
                "type": "object",
                "properties": {
                    "state": {
                        "type": "string",
                        "enum": ["pending", "started", "completed", "failed"]
                    },
                    "submitted": { "type": "integer", "description": "URLs submitted to scraper." },
                    "valid": { "type": "integer", "description": "Valid URLs after validation." },
                    "succeeded": { "type": "integer", "description": "Successfully scraped URLs." },
                    "failed": { "type": "integer", "description": "Failed scrapes." },
                    "started_at": { "type": "integer", "nullable": true },
                    "completed_at": { "type": "integer", "nullable": true },
                    "error": { "$ref": "#/components/schemas/StepError" }
                }
            },
            "AnalyzingStatus": {
                "type": "object",
                "description": "Gap analysis phase - determines what needs LLM enrichment.",
                "properties": {
                    "state": {
                        "type": "string",
                        "enum": ["pending", "started", "completed", "failed"]
                    },
                    "items_analyzed": { "type": "integer" },
                    "items_needing_enrichment": { "type": "integer" },
                    "items_complete": { "type": "integer", "description": "Items with all fields present from parser." },
                    "fields_to_enrich": {
                        "type": "object",
                        "nullable": true,
                        "description": "Count of items needing each field.",
                        "properties": {
                            "page_type": { "type": "integer" },
                            "author": { "type": "integer" },
                            "date_published": { "type": "integer" },
                            "date_modified": { "type": "integer" },
                            "brand_mentions": { "type": "integer" }
                        }
                    },
                    "started_at": { "type": "integer", "nullable": true },
                    "completed_at": { "type": "integer", "nullable": true }
                }
            },
            "EnrichingStatus": {
                "type": "object",
                "description": "LLM extraction phase via Batch API.",
                "properties": {
                    "state": {
                        "type": "string",
                        "enum": ["pending", "started", "completed", "failed", "skipped"]
                    },
                    "items_to_enrich": { "type": "integer" },
                    "items_enriched": { "type": "integer" },
                    "items_failed": { "type": "integer" },
                    "llm_batch_id": { "type": "string", "nullable": true, "description": "LLM Batch API ID" },
                    "batch_mode": { "type": "boolean", "description": "Whether async batch API was used" },
                    "llm": { "$ref": "#/components/schemas/LLMInfo" },
                    "started_at": { "type": "integer", "nullable": true },
                    "completed_at": { "type": "integer", "nullable": true },
                    "error": { "$ref": "#/components/schemas/StepError" }
                }
            },
            "LLMInfo": {
                "type": "object",
                "nullable": true,
                "description": "LLM provider and usage details.",
                "properties": {
                    "provider": { "type": "string", "description": "Processing provider" },
                    "model": { "type": "string", "description": "Model used for extraction" },
                    "usage": {
                        "type": "object",
                        "properties": {
                            "prompt_tokens": { "type": "integer" },
                            "completion_tokens": { "type": "integer" },
                            "total_tokens": { "type": "integer" }
                        }
                    },
                    "completion_time_ms": { "type": "integer", "nullable": true, "description": "Processing time in ms" }
                }
            },
            "StepError": {
                "type": "object",
                "nullable": true,
                "properties": {
                    "code": { "type": "string" },
                    "message": { "type": "string" }
                }
            },
            "ValidationError": {
                "type": "object",
                "properties": {
                    "error_code": { "type": "string", "example": "validation_error" },
                    "error_message": { "type": "string" },
                    "details": {
                        "type": "array",
                        "items": {
                            "type": "object",
                            "properties": {
                                "path": { "type": "string" },
                                "message": { "type": "string" },
                                "code": { "type": "string" }
                            }
                        }
                    }
                }
            },
            "Error": {
                "type": "object",
                "properties": {
                    "error_code": { "type": "string" },
                    "error_message": { "type": "string" }
                }
            },
            "Metadata": {
                "type": "object",
                "description": "Key-value pairs for custom data. Max 50 keys, key max 40 chars, value max 500 chars.",
                "additionalProperties": {
                    "type": "string",
                    "maxLength": 500
                },
                "maxProperties": 50,
                "example": {
                    "project_id": "12345",
                    "source": "weekly-crawl"
                }
            }
        }
    }
}