# pronto.stream
#
# The sitemap below is an index. Its children are sitemap-core.xml and
# sitemap-hubs/N.xml.

User-agent: *
Allow: /
Allow: /seo/
Allow: /mcp
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /openapi.json
Allow: /server.json
Allow: /terms
Allow: /.well-known/

# The authenticated API. Nothing here is indexable and every path answers 401 to
# an anonymous fetch, so crawling it spends budget to collect errors.
Disallow: /v1/
Disallow: /api/

# ==============================================================================
# 1. Frontier AI Training & Evaluation Crawlers (US & Global)
# ==============================================================================
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: GrokBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: CCBot
Allow: /

User-agent: Diffbot
Allow: /

# ==============================================================================
# 2. Real-Time AI Search, Citation & User-Browsing Agents (US & Global)
#
# xAI publishes several tokens and reporting on which one actually arrives is
# inconsistent, so all of the documented spellings are named rather than one.
# A robots record costs a line; a crawler that finds no record for itself and
# falls through to `*` is indistinguishable, in our own logs, from one that
# never came. What settles it is the User-Agent recorded at the edge — see
# `crawlerAgents` in internal/service/analytics/service.go, which counts them.
# ==============================================================================
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Grok-DeepSearch
Allow: /

User-agent: xAI-Bot
Allow: /

User-agent: Grok
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: YouBot
Allow: /

# ==============================================================================
# 3. Chinese Frontier AI Crawlers & Real-Time Search Engines
# ==============================================================================
User-agent: DeepSeekBot
Allow: /

User-agent: deepseek-crawler
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Baiduspider-render
Allow: /

User-agent: QwenBot
Allow: /

User-agent: KimiBot
Allow: /

User-agent: MoonshotBot
Allow: /

User-agent: ChatGLM
Allow: /

User-agent: GLM-Crawler
Allow: /

User-agent: MiniMaxBot
Allow: /

User-agent: Sogou spider
Allow: /

User-agent: 360Spider
Allow: /

User-agent: YisouSpider
Allow: /

# ==============================================================================
# 4. European & Independent Alternative AI / Search Engines
# ==============================================================================
User-agent: MistralAI
Allow: /

User-agent: MistralBot
Allow: /

User-agent: AlephAlphaBot
Allow: /

User-agent: MojeekBot
Allow: /

User-agent: Qwantify
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: BraveBot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: SeekportBot
Allow: /

Sitemap: https://pronto.stream/sitemap.xml