# Qenzo — https://qenzo.ai
#
# AI crawlers are welcome here, and that is a deliberate decision rather than
# an oversight: people increasingly ask an assistant "what should I use to
# dictate messages", and a site that blocks those crawlers cannot be the
# answer. The named agents below are listed explicitly because a blanket
# Allow is not always honoured by them.

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

# Claude-SearchBot is the third of Anthropic's three agents (the one that
# indexes for search quality) and was the only one relying on the wildcard
# below while its two siblings were named. Named agents are honoured more
# reliably than a wildcard, which is the whole reason this list exists.
User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: CCBot
Allow: /

# Yandex isn't "just another search engine": the product has a Russian- and
# Armenian-speaking audience, and they search there, not on Google.
User-agent: YandexBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Meta names two crawlers: Meta-ExternalAgent above (training) and this one,
# which is the one that feeds search.
User-agent: meta-webindexer
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: *
Allow: /
# Signed-in surfaces: nothing useful to a crawler, and no reason to have them
# indexed. /search, /insights and /style render an empty shell without a token
# in exactly the same way /account does, and were missing from this list.
Disallow: /account
Disallow: /admin
Disallow: /reset
Disallow: /search
Disallow: /insights
Disallow: /style
# API endpoints, not pages: there's nothing under them worth showing in
# search results, but crawling them costs us junk in the indexing report.
# On 24.08 Search Console showed 8 "other 4xx" and 5 "401" -- that was the
# crawler hitting exactly these, plus a separate 404 on /auth/reset (the
# reset page lives at /reset, while /auth is the router prefix, see
# auth_router.py).
Disallow: /auth
Disallow: /billing
Disallow: /v1

Sitemap: https://qenzo.ai/sitemap.xml