# CarScout robots.txt
# https://usecarscout.com

User-agent: *
Allow: /
Allow: /tools/
Allow: /data

# Authenticated/private routes
Disallow: /api/
Disallow: /matches
Disallow: /scouts/new
Disallow: /scouts/*/edit
Disallow: /alerts
Disallow: /settings
Disallow: /billing
Disallow: /checkout

# Block URL parameters to prevent crawl budget waste
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?page=
Disallow: /*?utm_*
Disallow: /*?ref=
Disallow: /*?source=

# AI retrieval bots (allowed, these cite sources in answers)
User-agent: GPTBot
Allow: /
Allow: /market
Allow: /tools/
Allow: /data
Disallow: /api/
Disallow: /matches
Disallow: /scouts/new
Disallow: /scouts/*/edit
Disallow: /alerts
Disallow: /settings
Disallow: /billing
Disallow: /checkout

User-agent: ClaudeBot
Allow: /
Allow: /market
Allow: /tools/
Allow: /data
Disallow: /api/
Disallow: /matches
Disallow: /scouts/new
Disallow: /scouts/*/edit
Disallow: /alerts
Disallow: /settings
Disallow: /billing
Disallow: /checkout

User-agent: PerplexityBot
Allow: /
Allow: /market
Allow: /tools/
Allow: /data
Disallow: /api/
Disallow: /matches
Disallow: /scouts/new
Disallow: /scouts/*/edit
Disallow: /alerts
Disallow: /settings
Disallow: /billing
Disallow: /checkout

# AI training-only bots (blocked, defense in depth with WAF)
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Sitemap
Sitemap: https://usecarscout.com/sitemap.xml

# LLM context files
# https://usecarscout.com/llms.txt
# https://usecarscout.com/llms-full.txt