User-Agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Allow: /web-scraping-api
Allow: /features
Allow: /features/
Allow: /platforms
Allow: /platforms/
Allow: /use-cases
Allow: /use-cases/
Allow: /compare
Allow: /compare/
Allow: /sdk
Allow: /sdk/
Allow: /integrations
Allow: /integrations/
Allow: /mcp
Allow: /blog
Allow: /blog/
Allow: /pricing
Allow: /terms
Allow: /privacy
Allow: /acceptable-use
Allow: /docs
Allow: /docs/
Allow: /playground
Allow: /playground/
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

User-Agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /dashboard/
Disallow: /account/
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /internal/
Disallow: /login
Disallow: /signin
Disallow: /signup
Disallow: /activate
Disallow: /activate/

Host: https://crawlora.net
Sitemap: https://crawlora.net/sitemap.xml