Olostep + Mastra Integration

npm install @olostep/mastra-tools @mastra/core

import { Mastra } from '@mastra/core';
import { createOlostepIntegration } from '@olostep/mastra-tools';

// Create the Olostep integration
const olostep = createOlostepIntegration();

// Register APIs (this makes them available to agents)
olostep.registerApis();

// Add to your Mastra config
export const mastra = new Mastra({
  config: {
    integrations: [olostep],
    // ... other config
  },
});

export OLOSTEP_API_KEY=your-api-key-here

OLOSTEP_API_KEY=your-api-key-here

// In your agent or workflow
const result = await mastra.callApi({
  integrationName: 'olostep',
  api: 'scrapeWebsite',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      url_to_scrape: 'https://example.com',
      formats: ['markdown'],
      country: 'US',
    }
  }
});

const result = await mastra.callApi({
  integrationName: 'olostep',
  api: 'batchScrape',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      batch_array: [
        { url: 'https://example.com', custom_id: 'site1' },
        { url: 'https://test.com', custom_id: 'site2' },
      ],
      formats: ['markdown'],
    }
  }
});

const result = await mastra.callApi({
  integrationName: 'olostep',
  api: 'createCrawl',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      start_url: 'https://docs.example.com',
      max_pages: 50,
      follow_links: true,
      formats: ['markdown'],
    }
  }
});

const result = await mastra.callApi({
  integrationName: 'olostep',
  api: 'createMap',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      url: 'https://example.com',
      search_query: 'blog',
      top_n: 100,
      include_urls: ['/blog/**'],
    }
  }
});

import { Agent } from '@mastra/core';
import { createOlostepIntegration } from '@olostep/mastra-tools';

const olostep = createOlostepIntegration();
olostep.registerApis();

const agent = new Agent({
  name: 'web-researcher',
  instructions: `
    You are a web research assistant. When users ask you to get information from a website,
    use the Olostep scrapeWebsite API to extract the content, then summarize it for them.
  `,
  model: 'openai/gpt-4',
});

// The agent can now use Olostep APIs through Mastra's API system

// 1. Map a website to discover URLs
const mapResult = await mastra.callApi({
  integrationName: 'olostep',
  api: 'createMap',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      url: 'https://example.com',
      include_urls: ['/blog/**'],
    }
  }
});

// 2. Batch scrape discovered URLs
const batchResult = await mastra.callApi({
  integrationName: 'olostep',
  api: 'batchScrape',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      batch_array: mapResult.urls.slice(0, 10).map(url => ({ url })),
      formats: ['markdown'],
    }
  }
});

// 3. Process results with your agent
const summary = await agent.generate({
  messages: [{
    role: 'user',
    content: `Summarize this content: ${batchResult.result.markdown_content}`
  }]
});

Agent Workflow:
1. Scrape product pages (scrapeWebsite)
2. Extract structured data (with parser)
3. Track price changes
4. Generate alerts
5. Update database

Agent Workflow:
1. Map website structure (createMap)
2. Crawl important sections (createCrawl)
3. Analyze content quality
4. Identify SEO opportunities
5. Generate recommendations

const result = await mastra.callApi({
  integrationName: 'olostep',
  api: 'scrapeWebsite',
  payload: {
    data: {
      apiKey: process.env.OLOSTEP_API_KEY,
      url_to_scrape: 'https://www.amazon.com/dp/PRODUCT_ID',
      formats: ['json'],
      parser: '@olostep/amazon-product',
    }
  }
});

import { Mastra } from '@mastra/core';
import { Agent } from '@mastra/core';
import { createOlostepIntegration } from '@olostep/mastra-tools';

// Create and register Olostep integration
const olostep = createOlostepIntegration();
olostep.registerApis();

// Initialize Mastra
export const mastra = new Mastra({
  config: {
    integrations: [olostep],
    // ... other config
  },
});

// Create research agent
const researchAgent = new Agent({
  name: 'research-assistant',
  instructions: `
    You are a research assistant that can search, extract, and structure web data.
    When users ask you to research a topic:
    1. Use Olostep's createMap to discover relevant pages
    2. Use batchScrape to extract content from multiple sources
    3. Analyze and summarize the findings
    4. Present structured research reports
  `,
  model: 'openai/gpt-4',
});

// Use the agent
async function researchTopic(topic: string) {
  // Step 1: Discover relevant pages
  const mapResult = await mastra.callApi({
    integrationName: 'olostep',
    api: 'createMap',
    payload: {
      data: {
        apiKey: process.env.OLOSTEP_API_KEY!,
        url: `https://example.com/search?q=${topic}`,
        top_n: 20,
      }
    }
  });

  // Step 2: Scrape discovered pages
  const batchResult = await mastra.callApi({
    integrationName: 'olostep',
    api: 'batchScrape',
    payload: {
      data: {
        apiKey: process.env.OLOSTEP_API_KEY!,
        batch_array: mapResult.urls.slice(0, 10).map(url => ({ url })),
        formats: ['markdown'],
      }
    }
  });

  // Step 3: Analyze with agent
  const summary = await researchAgent.generate({
    messages: [{
      role: 'user',
      content: `Based on this research data, provide a comprehensive summary of ${topic}`
    }]
  });

  return summary;
}