从Stripe网站提取博客URL

import requests
import time
import json

# 配置
API_URL = 'https://api.olostep.com/v1'
API_KEY = '<your_olostep_api_key>'
HEADERS = {
    'Content-Type': 'application/json',
    'Authorization': f'Bearer {API_KEY}'
}

# 开始时间，用于延迟跟踪
start_time = time.time()

# 定义包含URL模式的负载
payload = {
    "url": "https://stripe.com",
    "include_urls": ["/blog", "/blog/**"]  # 匹配 /blog 及其下的所有路径
}

# 发起请求
response = requests.post(f'{API_URL}/maps', headers=HEADERS, json=payload)

# 计算延迟
latency = round((time.time() - start_time) * 1000, 2)
print(f"请求完成耗时 {latency}ms")

# 处理结果
data = response.json()
print(f"在Stripe网站上找到 {data['urls_count']} 个博客URL")

# 打印前10个URL作为示例
print("\n示例博客URL:")
for url in data['urls'][:10]:
    print(f"- {url}")
    
# 将博客URL保存到文件以供进一步处理
with open('stripe_blog_urls.json', 'w') as f:
    json.dump(data, f, indent=2)
print(f"\n所有博客URL已保存到 stripe_blog_urls.json")

# 定义更具体的URL模式负载
payload = {
    "url": "https://stripe.com",
    "include_urls": ["/blog/engineering", "/blog/engineering/**"]
}