RAG Knowledge Base Building

const axios = require('axios');

const API_KEY = 'your_api_key';

// Extract documentation content for RAG
const docUrl = 'https://docs.example.com/api/authentication';
const content = await axios.get('https://api.webscraping.ai/ai/fields', {
  params: {
    api_key: API_KEY,
    url: docUrl,
    fields: JSON.stringify({
      title: 'Page title',
      main_content: 'Main content text without navigation',
      sections: 'Array of section headings and their content',
      code_examples: 'Any code snippets on the page',
      key_concepts: 'Key concepts or terms defined',
      related_topics: 'Links to related documentation pages'
    })
  }
});

console.log(content.data);
// {
//   "title": "API Authentication Guide",
//   "main_content": "This guide covers authentication methods...",
//   "sections": [
//     {"heading": "API Keys", "content": "API keys are..."},
//     {"heading": "OAuth 2.0", "content": "For OAuth flow..."}
//   ],
//   "code_examples": ["curl -H 'Authorization: Bearer...'"],
//   "key_concepts": ["API key", "Bearer token", "OAuth scope"],
//   "related_topics": ["/docs/rate-limits", "/docs/errors"]
// }

// Generate a summary for the knowledge base
const summary = await axios.get('https://api.webscraping.ai/ai/question', {
  params: {
    api_key: API_KEY,
    url: docUrl,
    question: 'Provide a 2-3 sentence summary of this page suitable for a knowledge base index.'
  }
});