User-agent: *
Disallow: /api
Disallow: /description.md
Disallow: /openapi.yml
Disallow: /email-confirmation
Sitemap: https://s3.amazonaws.com/webscraping-ai-sitemap/sitemap.xml.gz
Sitemap: https://s3.amazonaws.com/webscraping-ai-sitemap/sitemap1.xml.gz
Sitemap: https://webscraping.ai/sitemap.xml
Sitemap: https://webscraping.ai/sitemap1.xml
Sitemap: https://webscraping.ai/blog/sitemap.xml