# Celedog.io robots.txt
#
# Default policy for every crawler. We allow the marketing surfaces and
# block auth / account / admin / relay endpoints. Authenticated routes
# would 401 or 403 anyway, but disallowing them keeps search-engine
# crawl budget focused on indexable pages.
User-agent: *
Allow: /
Allow: /models
Allow: /pricing
Allow: /docs
Allow: /status
Allow: /about
Allow: /rankings
Allow: /privacy-policy
Allow: /user-agreement

Disallow: /auth
Disallow: /sign-in
Disallow: /sign-up
Disallow: /forgot-password
Disallow: /reset
Disallow: /otp
Disallow: /oauth
Disallow: /dashboard
Disallow: /keys
Disallow: /wallet
Disallow: /profile
Disallow: /usage-logs
Disallow: /subscriptions
Disallow: /admin
Disallow: /setup
Disallow: /cart

# API + relay endpoints, never indexable.
Disallow: /api/
Disallow: /v1/
Disallow: /v1beta/

# ----------------------------------------------------------------------
# AI / LLM crawlers
#
# We *want* the model directory, pricing, and docs to be discoverable by
# Claude, ChatGPT browsing, Perplexity, You.com, and similar: that's where a
# developer searching "what's the cheapest gateway to GPT-5" or "Claude
# 4.6 pricing" should find us. The same auth/admin/API disallow list
# above still applies; these blocks just confirm the AI crawlers are
# welcome on the public marketing surfaces.
#
# Crawler IDs sourced from each vendor's published documentation as of
# 2026-05. Anyone we haven't listed falls through to the User-agent: *
# block above, which already allows the marketing pages.

User-agent: GPTBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /keys
Disallow: /wallet
Disallow: /profile
Disallow: /usage-logs
Disallow: /subscriptions
Disallow: /admin
Disallow: /setup
Disallow: /cart
Disallow: /api/
Disallow: /v1/
Disallow: /v1beta/

User-agent: OAI-SearchBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/
Disallow: /v1/

User-agent: ChatGPT-User
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/
Disallow: /v1/

User-agent: ClaudeBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /keys
Disallow: /wallet
Disallow: /profile
Disallow: /admin
Disallow: /api/
Disallow: /v1/

User-agent: Claude-Web
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: DuckAssistBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: MistralAI-User
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

User-agent: YouBot
Allow: /
Disallow: /auth
Disallow: /dashboard
Disallow: /admin
Disallow: /api/

Sitemap: https://celedog.io/sitemap.xml